This is an automated email from the ASF dual-hosted git repository.
pifta pushed a change to branch HDDS-5447-httpfs
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 6185f64aab HDDS-8045. Dependency convergence error for zookeeper
(#4329)
add 9b48d8b1c0 HDDS-7696. MisReplicationHandler does not consider
QUASI_CLOSED replicas as sources (#4144)
add 41f048804a HDDS-7339. Implement Certificate renewal task for services
(#3982)
add 10811c597f HDDS-7666. EC: Unrecoverable EC containers with some
remaining replicas may block decommissioning (#4118)
add 9d30c9a7ce HDDS-7606. Add SCM HA support in intellij run (#4058)
add 7d6ee034d7 HDDS-7727. EC: SCM unregistered event handler for
DatanodeCommandCountUpdated (#4147)
add ecc2f189e4 HDDS-7708. No check for certificate duration config
scenarios. (#4149)
add f2f0afa9f6 HDDS-7243. Remove RequestFeatureValidator from echoRPC
method which supports only ValidationCondition.OLDER_CLIENT_REQUESTS (#4051)
add 2eb5805b6c HDDS-7738. SCM terminates when adding container to a closed
pipeline (#4154)
add e09f9a208a HDDS-7729. EC: ECContainerReplicaCount should handle
pending delete of unhealthy replicas (#4146)
add 4abe983acf HDDS-7695. EC metrics related to replication commands don't
add up (#4152)
add a1cb9d567d HDDS-7628. Intermittent failure in
TestOzoneContainerWithTLS (#4142)
add dcfd621e15 HDDS-7652. Volume Quota not enforced during write when
bucket quota is not set (#4124)
add 07b67597e8 HDDS-7739. EC: Increase the information in the RM sending
command log message (#4153)
add a80c6b1c73 HDDS-7726. EC: Enhance datanode reconstruction log message
(#4155)
add b6cda5162f HDDS-4539. Container Health Task should not run until Recon
has reached steady state. (#4049)
add ed7c60cafd HDDS-7560. Placement Policy Interface changes to handle
Overreplication (#4014)
add ab91e46247 HDDS-7638. Ozone client change to support HSync. (#4104)
add cbbddc6fdc HDDS-7630. Show detailed OMResponse when
OzoneManagerDoubleBuffer terminates (#4075)
add dff829623b HDDS-7705. Fix OM Bootstrap request (#4160)
add 4531701c4d HDDS-7753. Simplify DatanodeDetails#toString to improve log
messages (#4161)
add de53086f13 HDDS-5541. Namespace summary endpoint can carry basic
information on the entity as well. (#4042)
add 7229b35958 HDDS-6447. Refine SCM handling of unhealthy container
replicas. (#3920)
add d56acae8c8 HDDS-7760. Let ContainerDataConstructor extend
SafeConstructor (#4162)
add 9d5cfd6934 HDDS-7083. Spread container-copy directories (#3648)
add 5f54cc60ba HDDS-5934. Add OM NodeID to VERSION file. (#4138)
add 2bd13c3737 HDDS-7751. Volume quota must have all bucket quota set
(#4165)
add 80f544b704 HDDS-7570. Provide a shareable ofs://temp directory (#4027)
add 1a63207add HDDS-7775. EC: Exception encountered while deleting
UNHEALTHY replica in Datanode (#4173)
add f9dfd73e99 HDDS-7761. EC: ReplicationManager - Use
placementPolicy.replicasToRemoveToFixOverreplication in EC Over replication
handler (#4166)
add 253e97c218 HDDS-7774. Update outdated Trash documentation (#4172)
add c00d3af98a HDDS-7722. FSO buckets fail to invalidate open file table
cache when committing a key (#4156)
add 7923e25be4 HDDS-7441. Rename function names of retrieving metadata
keys (#3918)
add 59938f90bf HDDS-7455. ClassCastException: OzoneTokenIdentifier cannot
be cast to String (#4159)
add 8b9aad6a45 HDDS-7754. Download of container is failing with SSL/TLS
error during re-replication (#4174)
add c61ee3ea3e HDDS-7787. GetChecksum for EC files can fail intermittently
with IndexOutOfBounds exception (#4180)
add b405f84621 HDDS-7542. Refactor DefaultReplicationConfig (#4005)
add 0c32c60257 HDDS-7718. Bump Netty to 4.1.86 and gRPC to 1.51.1 (#4139)
add 3bc2b79764 HDDS-7788. Ratis OverReplicationHandler should exclude
stale replicas (#4183)
add dbe9abc670 HDDS-7723. Refresh Keys and Certificate used in
OzoneSecretManager after certificate renewed (#4179)
add d3f232396a HDDS-7748. Rename OMFileRequest.addToOpenFileTable() to
avoid misuse. (#4176)
add be408031e3 HDDS-7804. UNHEALTHY replicas will not contribute to
sufficient replication in RatisContainerReplicaCount (#4192)
add 6ff310eedd HDDS-7755. Ensure that acquired locks are always released.
(#4191)
add 099cd1d680 HDDS-7807. TarContainerPacker closes streams multiple times
(#4193)
add c6c36d4bcb HDDS-7801. Bucket not found when calling getKeyInfo with
tenant context (#4189)
add 23a91637ff HDDS-7625. Do not compress OM/SCM checkpoints (#4130)
add 5ea64550b5 HDDS-7813. Handle Mismatched Replicas (OPEN or CLOSING) of
QUASI-CLOSED containers in RM (#4195)
add 61a2e6e4b8 HDDS-7097. Container scanner log output lacks useful
information (#4169)
add c25d38faf8 HDDS-7792. Fix package name typos in
o.a.h.hdds.security.x509 (#4184)
add aed64de760 HDDS-7777. Implement container replication in push model
(#4197)
add 151ee0afd8 HDDS-7820. Store container descriptor at start of tar
archive (#4200)
add f1e94fc801 HDDS-7825. Warn when EC write exception occurs (#4201)
add 43b01d1bc5 HDDS-7833. EC: Refactor ReplicationSupervisor to allow
Replication and ECReconstruction tasks (#4202)
add 08d8521060 HDDS-7822. Allow multiple commands per datanode in
UnhealthyReplicationHandler (#4203)
add 0217fa4e8e HDDS-7834. Remove unnecessary streams (#4208)
add 4c6c1218a7 HDDS-7818. Modify Ratis Replication Handling in the new RM
(#4204)
add e806fd6ce8 HDDS-7496. Make default bucket layout FSO in a backwards
compatible way. (#3966)
add 17549db4a1 HDDS-7841. EC: Remove ECReconstructionSupervisor and send
reconstruction commands to ReplicationSupervisor (#4210)
add ec3813fada HDDS-7808. Intermittent failure in
TestReplicationManager#testUnderReplicationQueuePopulated (#4213)
add ec1e098694 HDDS-7372. Fix missing jars in classpath by specifying jar
versions (#4097)
add 75a28b2e4c HDDS-7806. Add unit tests for push replication (#4216)
add 38733cfff1 HDDS-7770. Recon namespace summary endpoint to carry basic
entity information as well (#4170)
add 9d4d358654 HDDS-7778. Add metrics for push replication (#4217)
add 4eddb9d2ac HDDS-6847. Audit for ALLOCATE_BLOCK should indicate number
of allocated blocks (#3518)
add 3393354472 HDDS-7856. Disable flaky TestPushReplicator until fixed
add b16342af26 HDDS-7844. EC: Add normal and low priority to replication
supervisor and commands (#4220)
add 6389d01605 HDDS-7856. Fix timeout in TestPushReplicator (#4225)
add 3084e8d41e HDDS-7859. ICR processing does not remove container
reference in NodeManager for a deleted replica (#4228)
add 1a796f1b54 HDDS-7821. Let push replication use compression from
configuration (#4229)
add 2410551d2e HDDS-6849. OM changes for tracking Snapshot Metadata.
(#3582)
add a0a9d221d0 Merge branch 'master' into HDDS-6517-Snapshot
add 4dba9020ed HDDS-6853. [Snapshot] Implement Create Snapshot API. (#3652)
add eafd2ccec0 HDDS-6964. [Snapshot] Split out shared "Path based access"
code from OM. (#3653)
add 03dfa243ac HDDS-7224. Create a new RocksDBCheckpoint Diff utility.
(#3755)
add 6eb19a7710 HDDS-6855. [SNAPSHOTS] Path Based Access for Ozone
Snapshots. (#3729)
add 4be3864675 Merge branch 'master' into HDDS-6517-Snapshot
add f30c0621ea HDDS-6983. Snapshot Chain - list of snapshots per
snapshottable bucket (#3658)
add 9538954e4b HDDS-7427. [Snapshot] Add unit-testcases for Ozone snapshot
feature. (#3896)
add e642ddeb55 HDDS-7281. [Snapshot] Handle RocksDB compaction DAG
persistence and reconstruction (#3824)
add d1a834ce09 HDDS-7452. [snapshot] Add unit-testcases for snapshot
create validation. (#3946)
add f1f1c0ba42 HDDS-7375. [Snapshot] Implement `ozone fs` command for
creating snapshot (#3937)
add 83072e0f8d HDDS-7410. Ozone snapshot diff skeleton code. (#3885)
add 06ae2fe8d5 HDDS-7494. [Snapshot] Fix SnapshotInfo#dbTxSequenceNumber
(de)serialization (#3965)
add 67e67250bd HDDS-6854. [Snapshot] Implement List Snapshot API (#3784)
add 6fe2ab8011 HDDS-7279. Snapshot Create requires Double Buffer Flush
thread to split the commit batch. (#3958)
add 1a52ae199a HDDS-7571. [snapshot] Add unit-testcases for Ozone Snapshot
create API name validation. (#4023)
add d96f207f3a HDDS-7512. [snapshot] List Snapshot returns an empty list
for a non-existent bucket (#3993)
add f77dfa6f59 HDDS-6962. [Snapshot] Background Service to delete
irrelevant SST files in a snapshot. (#3883)
add e5d9c38f6f HDDS-7466. SnapshotDiff Client Code. (#3942)
add 2554ec26d7 HDDS-7508. [Snapshot] Finish TestOMSnapshotDAG, restore
CompactionNode SnapshotID field for debugging (#3981)
add 1ee083178d HDDS-7484. Snapshot ID on followers should match the one on
the OM Leader (#3985)
add 113072c16c HDDS-7583. Support efficient Snapdiff only until configured
Snapshot history (#4036)
add 7f8838070e HDDS-7647. [snapshot] Add unit-testcases for Ozone Snapshot
List API (#4090)
add 96bb4369c0 Revert "HDDS-7647. [snapshot] Add unit-testcases for Ozone
Snapshot List API (#4090)" (#4102)
add 39bd6090bd Revert "HDDS-7583. Support efficient Snapdiff only until
configured Snapshot history (#4036)" (#4103)
add 3d289e9050 HDDS-7647. [snapshot] Add unit-testcases for Ozone Snapshot
List API (#4105)
add 9fe6d106ac HDDS-7524. Compaction DAG node pruning (#4045)
add 125eccfef2 HDDS-7607. [Snapshot] SnapshotDiff command lists unmodified
file as modified (#4108)
add 0bcd697107 HDDS-7690. [Snapshot] Use SST file list output from
compaction DAG as SnapshotDiff input (#4119)
add d8765436c2 Merge remote-tracking branch 'asf/master' into
HDDS-6517-Snapshot
add 98e3a5033d Merge remote-tracking branch 'asf/master' into
HDDS-6517-Snapshot
add 52fb8ba492 HDDS-7690. Addendum: [Snapshot] Use SST file list output
from compaction DAG as SnapshotDiff input (#4168)
add 52103f273a HDDS-6984. [Snapshot] Prevent bucket from being deleted if
snapshot exists (#4015)
add 0c25be8cdf HDDS-7702. [snapshot] Add unit-testcases for Ozone fs
createSnapshot (#4122)
add 26006d4fcf HDDS-7765. [Snapshot] Handle OzoneManager#getKeyInfo with
OmMetadataReader. (#4185)
add 01762a385d HDDS-7506. [Snapshot] Expose more snapshot metrics under
OMMetrics (#4164)
add c88dd820e1 HDDS-7779. [snapshot] Add unit-testcases for Ozone
SnapshotDiff.
add 3e31e047d9 HDDS-7768. [Snapshot] Add acceptance tests (#4171)
add dae9285af3 HDDS-7704. [snapshot] Add unit-testcases for snapshot
restore (#4148)
add 40688f637a HDDS-7819. [Snapshot] Add unit-testcases for fs delete of
bucket havng snapshots (#4198)
add ae83ac54e0 HDDS-7600. [Snapshot] Mark all new proto message fields as
optional (#4120)
add aea5edfe99 HDDS-7845. [Snapshot] Wait for RocksDB checkpoint directory
creation (#4214)
add 106b193c2a Merge remote-tracking branch 'asf/master' into
HDDS-6517-Snapshot
add ccc814ee7f HDDS-6857. [Snapshot] Implement Snapshot Delete CLI and API
(#4175)
add afa6030b90 HDDS-7861. [Snapshot] Delete keys from the source bucket to
fix the flakiness of snapshot restore tests (#4230)
add f01cebe19b Merge remote-tracking branch 'asf/HDDS-6517-Snapshot'
add 2eae7cd575 HDDS-7826. Add support for application to probe output
stream capability (#4205)
add b44a77db99 HDDS-7874. Disable flaky unit test:
TestHddsSecureDatanodeInit.testCertificateRotation
add ab721e8ceb HDDS-7850. Bump maven-enforcer-plugin version to 3.2.1
(#4218)
add 22f38d6f0c HDDS-5626. Mark parts of TestAddRemoveOzoneManager,
TestBlockOutputStreamFlushDelay and TestFailureHandlingByClient as flaky (#4240)
add a352ab9fcd HDDS-7688. Ozone Manager change to support HSync. (#4211)
add 583fdb8e34 HDDS-7879. Intermittent BindException in HA integration
tests (#4241)
add 8741083374 HDDS-7868. Intermittent failure in TestOmAcls (#4239)
add fd32156178 HDDS-4303. Ensure LayoutVersion is unique and monotonicaly
increasing in LayoutFeature enums (#4224)
add 9fba172b3f HDDS-7873. [Snapshot] Prune backup SST files that can be
expanded periodically (#4235)
add 3732f55c16 HDDS-7277. [Snapshot] Implement snapshot key listing in
Ozone Shell (#4125)
add 6a66db0a99 HDDS-7886. Mark parts of TestRatisPipelineCreateAndDestroy
as flaky
add 15cc89b602 HDDS-3591. ozone.administrators support dynamic changes
through the cli. (#4221)
add c6c58ab736 HDDS-7450. Improve bucket creation message (#3997)
add 4ded8faff3 HDDS-7881. Intermittent timeout in basic acceptance test
(#4243)
add 19e6ac9fb8 HDDS-7783. Add compile platform info in 'ozone version'
command output (#4177)
add 406aef41e0 HDDS-7747. Document jq filtering examples for CLI
responses. (#4157)
add 0a377c685a HDDS-7838. gRPC channel created block input/output stream
not shutdown properly (#4215)
add fda247cf51 HDDS-7306. DeleteBlock retry count increased at incorrect
place (#3818)
add 2e49013db4 HDDS-7091. Filter containers with pending deletion blocks
before checking policy (#3655)
add 6c44169771 HDDS-7863. Remove OzoneStreamCapabilities (#4247)
add 940d678ef8 HDDS-7578. Wrong directory config used for SCM HA Ratis
Snapshot (#4030)
add db1ea10645 HDDS-7126. Clean deletedBlock records of deleted containers
(#3683)
add 471f1b2a93 HDDS-7082. Delete out of date audit logs (#3647)
add 9882cf7390 HDDS-7382. FSO bucket `rename` supports change modification
time of parent dir (#3953)
add 0530b6dc88 HDDS-7854. Recon NPE when trace enabled (#4222)
add 3ca0a982dc HDDS-7851. Apply flush and compact on all column families
(#4219)
add 9df3916614 HDDS-7864. Add integration test for replication (#4238)
add db896cd2ae HDDS-3265. Intermittent timeout in TestRatisPipelineLeader
(#4248)
add 19467c4987 HDDS-7764. Potential discrepancy of key creation time may
cause premature open key clean up. (#4242)
add 3e682b8470 HDDS-7489. Create a Freon tool to simulate datanodes (#4126)
add dba3ef9463 HDDS-7617. Mark TestECContainerRecovery as flaky
add 07629324c0 HDDS-6444. Apply HDDS-5875 to OMKeyCommitRequestWithFSO
(#3250)
add 8007630491 HDDS-7857. Apply custom RocksDB log configuration for all
schema versions (#4223)
add 1db8b735c4 HDDS-7321. Auto rocksDB small sst files compaction. (#3908)
add 11fd5b12fd HDDS-7917. EC: ECBlockInputStream should try spare replicas
on error (#4253)
add 10a1fa6158 HDDS-7921. Migrate TestKeyManagerImpl to JUnit5 (#4257)
add 2250a481d6 HDDS-7799. Add container count to datanode usage info
(#4209)
add 5daca052cc HDDS-6056. Recon /containers endpoint should return SCM
container data instead of OM container data. (#4158)
add 5952e4ea80 HDDS-7871. Fix false positive in
KeyManagerImpl#createFakeDirIfShould() (#4236)
add 3f7ded2a34 HDDS-7915. Force close QUASI_CLOSED replicas only when the
container is CLOSED in Legacy RM (#4259)
add d1319d8187 HDDS-7924. Remove non-conflicting transitive-only
dependency versions (#4261)
add 1bb74438fa HDDS-7815. Extract S3 secret layer (#4196)
add b5230a2756 HDDS-7923. [EC] Reconstruction is failing with
IndexOutOfBoundsException (#4258)
add 975e835fc6 HDDS-7928. EC: Change ContainerReplicaPendingOps to store
deadline rather than scheduled time (#4260)
add b3aac19f6b HDDS-7920. User cannot list their own volumes without
access to root volume (#4254)
add a5aa549dc8 HDDS-7829. SCM to reject adding container to closed
pipeline (#4212)
add 13a284834b HDDS-7944. Add dash-separated-options where missing (#3943)
add 7c2db17276 HDDS-7634. Recon: Show Datanode UUID on Pipeline page
(#4135)
add a8d20de041 HDDS-7513. [Snapshot] Support list snapshots using fs -ls
(#4134)
add 9b5b8d6b32 HDDS-7667. [Admin Doc] Observability: Add Grafana
integration (#4267)
add 469c034423 HDDS-7687. Support OM transfer Ratis leadership (#4265)
add 55cc72e377 HDDS-7617. Intermittent timeout in TestECContainerRecovery
(#4268)
add 22b6852d73 HDDS-7617. Remove flaky tag from TestECContainerRecovery
add e1d4127322 HDDS-7847. Handle Replication of Unhealthy Replicas in RM
(#4227)
add 491791c4d3 HDDS-7594. [FSO] Folders created through S3G are created on
file system as "files". (#4186)
add 4748fb9726 HDDS-7866. [Snapshot] Fix ConcurrentModificationException
in snapdiff API. (#4233)
add b6ad97fd52 HDDS-7962. Add configuration flag to enable
LegacyReplicationManager for RATIS containers (#4270)
add 5d5d370de3 HDDS-7919. EC: ECPipelineProvider.createForRead should
filter out dead replicas and sort replicas (#4277)
add e7d57b9f91 HDDS-7800. [Snapshot] Use persistent map and set to
generate snapshot diff report (#4190)
add 6f6a0571fc HDDS-7964. Enable ratis pre-vote in Ozone by default.
(#4272)
add 097594f06b HDDS-7967. Replace Hadoop annotations with Ozone-specific
ones (#4279)
add b3b0538597 HDDS-4223. Optimize the construction method of OzoneVolume
and OzoneBucket. (#4262)
add 4811cd51d6 HDDS-7865. [Snapshot] Add test for getFileStatus and
readFile from snapshot paths. (#4232)
add 3775c4de2b HDDS-7832. Support FSO buckets for FileSizeCountTask.
(#4206)
add fc8f79524f HDDS-7931. EC: ManagedChannelImpl not cleaned up properly
(#4269)
add bd900ae9db HDDS-7969. CacheValue should not store value as an
Optional. (#4275)
add c23ba9a116 HDDS-7556. Translate EC doc into Chinese (#4263)
add cdbe91dd8f HDDS-7980. Force Close QUASI_CLOSED replicas of CLOSED
containers in RM (#4283)
add 88c0a6621a HDDS-7976. FsShell creates unrelated trace spans (#4281)
add 23e94b66c0 HDDS-7379. Use certificate bundles instead of the sole
certificate (#4231)
add 4261c1df03 HDDS-6572. EC: ReplicationManager - add move manager for
container move (#4274)
add 43caa56e18 HDDS-7178. [Multi-Tenant] Use optimistic read in Ranger
background sync (#3725)
add 45362b15f0 HDDS-7461. Require CREATE right on volume instead of WRITE
for CREATE_BUCKET operation (#3938)
add 59c87ba995 HDDS-7301. Cleanup OmUtils (#3812)
add b466cd582e HDDS-7596. Fix OM crash due to a corner case for
FSO-enabled bucket (#4048)
add d2e3e7580d HDDS-7988. Run S3 tests with HA Proxy (#4284)
add 4be2122922 Revert "HDDS-7588. Intermittent failure in
TestObjectStoreWithLegacyFS#testFlatKeyStructureWithOBS (#4040)"
add 204f3b4b97 HDDS-7710. Support AWS s3 ListObjects API's encodingType
request parameter (#4127)
add 501152d313 HDDS-7588. Intermittent failure in
TestObjectStoreWithLegacyFS#testFlatKeyStructureWithOBS (#4290)
add cafb372799 HDDS-7064. S3 get-object response emits tracing spans
outside ObjectEndpoint#get (#4288)
add 330cec2797 HDDS-7971. Support SCM transfer Ratis leadership (#4276)
add 47a68f8751 HDDS-7989. UnhealthyReplicationProcessor retries failure
without delay (#4285)
add 4d09962a65 HDDS-7906. [Snapshot] Wait for checkpoint creation if
snapshot in cache and not committed to DB. (#4249)
add c9bbb03556 HDDS-6743. Specify leader node for OM failover (#3409)
add 06e7b1b616 HDDS-7721. Make OM Ratis roles available in /prom endpoint
(#4140)
add bcfebad032 HDDS-7721. Make OM Ratis roles available in /prom endpoint
(fix compile error)
add 170222e378 HDDS-7991. Do not return fake parent dir for deleted keys
(#4287)
add 6b1c62a88f HDDS-7993. [snapshot] Add testcase to handle snapshot name
minimum-length (#4293)
add 2fa1e5ab3b HDDS-7998. Synchronize on containerInfo in
ReplicationManager and MoveManager (#4295)
add 5da39f8cb8 HDDS-7839. Upgrade Weld to 3.1.9 (#4299)
add 19d9aa38c0 HDDS-7982. NPE in BlockInputStream due to null pipeline
after refresh (#4296)
add 73b339a55d HDDS-7983. Intermittent OutOfMemoryError in
TestOzoneRpcClientWithRatis#testUploadWithStreamAndMemoryMappedBuffer. (#4292)
add dabe58a2bc HDDS-7782. OM lease recovery for hsync'ed files. (#4255)
add c592d9c857 HDDS-8009. OM HA metrics should be unregistered if leader
is not known (#4300)
add 7da66e21ba HDDS-8012. Clean up after link loop test (#4304)
add ec2f36cbd0 HDDS-7973. Let RatisMisReplicationHandler use the new
RatisContainerReplicaCount constructor (#4305)
add 4465c494db HDDS-7890. Refactor ContainerDeletionChoosingPolicy
implementations (#4303)
add e3868fabdf HDDS-8020. File checksum helper leaking client (#4307)
add 0a66958755 HDDS-8025. ReplicationManager: Count a container once for
missing, under, mis or over replicated (#4313)
add aebc06f118 HDDS-8026. Replace import from shaded Guava (#4314)
add dd952d4aae HDDS-8008. Move pendingOps into ContainerStateManagerImpl
to ensure consistent state (#4298)
add ce57071a5f HDDS-7590. Use keyManager and trustManager provided by
keyStoreFactory in om grpc services (#4145)
add f88b9d0207 HDDS-8035. Mark TestOzoneManagerHAWithData#testOMHAMetrics
as flaky
add d75507d7ac HDDS-8035. Mark TestHSync#testO3fsHSync as flaky
add c69a015930 HDDS-8034. Check container replication health before
scheduling move in MoveManager (#4321)
add b858170a64 HDDS-8024. Mark TestHSync#testOfsHSync as flaky
add c755e5d645 HDDS-8037. Improve logging in EC Reconstruction putBlock
precondition check (#4320)
add e90e2dd8ea HDDS-7505. Recon: Rename OM DB Sync to DB Sync on UI (#3988)
add 628623d664 HDDS-6241. Follower SCM node repeatedly sending requests to
Ratis server. (#4302)
add b831d8e6fb HDDS-8036. Unprotected flush in SCMHADBTransactionBuffer
(#4319)
add 25ad1f2a4f HDDS-7183. Expose RocksDB critical properties. (#3875)
add a63be2662e HDDS-8044. Ensure GrpcOutputStream is closed (#4327)
add 0349c8e8a9 HDDS-8006. [snapshot] Add unit test cases for snapshot
delete (#4297)
add 1b18bd77b3 Revert "HDDS-8006. [snapshot] Add unit test cases for
snapshot delete (#4297)" (#4330)
add 9f77588334 HDDS-7959. Improve log in ECBlockInputStream and
ECBlockReconstructedStripeInputStream (#4323)
add 9e348de653 HDDS-7906. Addendum: [Snapshot] Wait for checkpoint
creation if snapshot in cache and not committed to DB. (#4306)
add abd9dd01f0 HDDS-534. Remove unsupported jmxget subcommand (#4326)
add f0c4d415c7 HDDS-7926. [hsync] Recon throws ClassCastException. (#4266)
add 3d648466ed HDDS-8032. SCM support reconfigurable dynamically (#4318)
add 575cf41950 HDDS-8006. [snapshot] Add unit test cases for snapshot
delete (#4331)
add 84f1523d24 HDDS-7869. Log configuration on component startup. (#4271)
add 488e1935a7 HDDS-7156. Reset pending delete block count (#4324)
add 3dd7dc4128 HDDS-8019. IllegalStateException: call already closed at
GrpcXceiverService.onCompleted (#4335)
new a142d70adb Merge branch 'master' into HDDS-5447-httpfs
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/hdds/scm/storage/BlockInputStream.java | 20 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 67 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 15 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 19 +-
.../ozone/client/io/BadDataLocationException.java | 32 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 110 +-
.../ozone/client/io/ECBlockInputStreamProxy.java | 9 +-
.../io/ECBlockReconstructedStripeInputStream.java | 42 +-
.../hdds/scm/storage/TestBlockInputStream.java | 94 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 3 +-
.../ozone/client/io/TestECBlockInputStream.java | 53 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 24 +
.../java/org/apache/hadoop/hdds/StringUtils.java | 13 +-
.../org/apache/hadoop/hdds/client/BlockID.java | 5 +
.../hadoop/hdds/client/ContainerBlockID.java | 3 +
.../hdds/client/DefaultReplicationConfig.java | 94 +-
.../hadoop/hdds/client/ECReplicationConfig.java | 7 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 10 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 4 +
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 64 +
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 3 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 10 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 8 +
.../hadoop/hdds/scm/container/ContainerInfo.java | 3 +
.../hadoop/hdds/scm/exceptions/SCMException.java | 2 +
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 2 +-
.../pipeline/DuplicatedPipelineIdException.java} | 24 +-
.../pipeline/InvalidPipelineStateException.java} | 25 +-
.../protocol/StorageContainerLocationProtocol.java | 8 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 6 +-
.../hadoop/hdds/security/x509/SecurityConfig.java | 45 +-
.../x509/certificate/utils/CertificateCodec.java | 226 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 49 +-
.../apache/hadoop/hdds/tracing/TracingUtil.java | 19 +
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 2 +
.../java/org/apache/hadoop/hdds/utils/IOUtils.java | 17 +
.../org/apache/hadoop/hdds/utils/VersionInfo.java | 4 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 32 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 11 +
.../org/apache/hadoop/ozone/audit/S3GAction.java | 3 +-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +-
.../main/resources/hdds-version-info.properties | 3 +-
.../common/src/main/resources/ozone-default.xml | 113 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 27 +
.../hadoop/ozone/audit/AuditLogTestUtils.java | 56 +-
.../hadoop/hdds/conf/ConfigurationTarget.java | 4 +
hadoop-hdds/container-service/pom.xml | 4 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 131 +-
.../container/common/helpers/ContainerUtils.java | 20 +
.../container/common/impl/ContainerDataYaml.java | 8 +-
.../RandomContainerDeletionChoosingPolicy.java | 56 +-
...TopNOrderedContainerDeletionChoosingPolicy.java | 63 +-
.../ContainerDeletionChoosingPolicyTemplate.java} | 57 +-
.../common/interfaces/ContainerPacker.java | 3 +-
.../common/statemachine/DatanodeConfiguration.java | 90 +-
.../common/statemachine/DatanodeStateMachine.java | 66 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 13 +-
.../ReconstructECContainersCommandHandler.java | 19 +-
.../ReplicateContainerCommandHandler.java | 24 +-
.../transport/server/GrpcXceiverService.java | 1 +
.../common/transport/server/XceiverServerGrpc.java | 2 +-
.../transport/server/ratis/XceiverServerRatis.java | 1 +
.../common/utils/db/DatanodeDBProfile.java | 1 +
.../ozone/container/common/volume/HddsVolume.java | 24 +
.../container/common/volume/StorageVolume.java | 4 +
.../ECReconstructionCommandInfo.java | 59 +-
.../ECReconstructionCoordinator.java | 59 +-
.../ECReconstructionCoordinatorTask.java | 105 +-
.../reconstruction/ECReconstructionSupervisor.java | 111 -
.../container/ec/reconstruction/TokenHelper.java | 16 +-
.../container/keyvalue/KeyValueContainer.java | 86 +-
.../container/keyvalue/KeyValueContainerCheck.java | 10 +-
.../container/keyvalue/KeyValueContainerData.java | 35 +-
.../KeyValueContainerMetadataInspector.java | 14 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +-
.../container/keyvalue/TarContainerPacker.java | 175 +-
.../helpers/KeyValueContainerLocationUtil.java | 41 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 10 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 14 +-
.../background/BlockDeletingService.java | 22 +-
.../container/metadata/AbstractDatanodeStore.java | 19 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 20 +-
.../ozone/container/metadata/DatanodeStore.java | 3 +
.../metadata/DatanodeStoreSchemaThreeImpl.java | 90 +
.../container/ozoneimpl/ContainerController.java | 3 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 5 +-
.../replication/AbstractReplicationTask.java | 123 +
.../container/replication/ContainerDownloader.java | 3 +-
.../container/replication/ContainerImporter.java | 127 +
.../replication/ContainerReplicationSource.java | 3 +-
.../container/replication/ContainerUploader.java | 20 +-
.../replication/CopyContainerCompression.java | 103 +-
.../replication/CopyContainerResponseStream.java | 47 +
.../replication/DownloadAndImportReplicator.java | 113 +-
.../replication/GrpcContainerUploader.java | 99 +
.../container/replication/GrpcOutputStream.java | 41 +-
.../replication/GrpcReplicationClient.java | 31 +-
.../replication/GrpcReplicationService.java | 31 +-
.../container/replication/MeasuredReplicator.java | 23 +-
.../OnDemandContainerReplicationSource.java | 20 +-
.../container/replication/PushReplicator.java | 82 +
.../container/replication/ReplicationServer.java | 7 +-
.../replication/ReplicationSupervisor.java | 170 +-
.../replication/ReplicationSupervisorMetrics.java | 7 +-
.../container/replication/ReplicationTask.java | 88 +-
.../replication/SendContainerOutputStream.java | 48 +
.../replication/SendContainerRequestHandler.java | 133 +
.../replication/SimpleContainerDownloader.java | 40 +-
.../protocol/commands/CloseContainerCommand.java | 10 +
.../protocol/commands/DeleteContainerCommand.java | 10 +
.../commands/ReconstructECContainersCommand.java | 15 +
.../commands/ReplicateContainerCommand.java | 84 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 223 +-
.../container/common/TestBlockDeletingService.java | 203 +-
.../TestSchemaOneBackwardsCompatibility.java | 27 +-
.../TestSchemaTwoBackwardsCompatibility.java | 4 +-
.../common/helpers/TestContainerUtils.java | 10 +
.../common/statemachine/TestStateContext.java | 9 +-
.../TestECReconstructionSupervisor.java | 158 -
.../container/keyvalue/TestKeyValueContainer.java | 146 +-
.../TestKeyValueContainerIntegrityChecks.java | 4 +-
.../TestKeyValueContainerMetadataInspector.java | 8 +-
.../container/keyvalue/TestTarContainerPacker.java | 90 +-
.../container/ozoneimpl/TestContainerReader.java | 13 +-
.../container/ozoneimpl/TestOzoneContainer.java | 7 +-
...OutputStream.java => GrpcOutputStreamTest.java} | 48 +-
.../ReplicationSupervisorScheduling.java | 85 +-
.../replication/TestCopyContainerCompression.java | 95 +
.../TestCopyContainerResponseStream.java | 50 +
.../replication/TestGrpcReplicationService.java | 72 +
.../replication/TestMeasuredReplicator.java | 32 +-
.../container/replication/TestPushReplicator.java | 164 +
.../replication/TestReplicationSupervisor.java | 242 +-
.../replication/TestSendContainerOutputStream.java | 75 +
.../replication/TestSimpleContainerDownloader.java | 56 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 23 +-
hadoop-hdds/docs/content/concept/Recon.md | 1 +
hadoop-hdds/docs/content/design/trash.md | 5 -
.../docs/content/feature/ErasureCoding.zh.md | 168 +
.../content/feature/GrafanaOzoneObjectMetrics.png | Bin 0 -> 319286 bytes
.../docs/content/feature/GrafanaOzoneOverall.png | Bin 0 -> 507556 bytes
.../content/feature/GrafanaOzoneRPCMetrics.png | Bin 0 -> 447935 bytes
hadoop-hdds/docs/content/feature/Observability.md | 42 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
hadoop-hdds/docs/content/feature/PrefixFSO.zh.md | 4 +-
.../docs/content/feature/Reconfigurability.md | 83 +
.../docs/content/feature/Reconfigurability.zh.md | 85 +
hadoop-hdds/docs/content/interface/Cli.md | 62 +
hadoop-hdds/docs/content/tools/LogsInOzone.md | 45 +
hadoop-hdds/docs/content/tools/_index.md | 1 -
hadoop-hdds/docs/content/tools/_index.zh.md | 1 -
.../apache/ozone/erasurecode/CodecRegistry.java | 2 +-
.../org/apache/ozone/erasurecode/package-info.java | 4 +-
.../rawcoder/AbstractNativeRawDecoder.java | 2 +-
.../rawcoder/AbstractNativeRawEncoder.java | 2 +-
.../rawcoder/ByteArrayEncodingState.java | 2 +-
.../rawcoder/ByteBufferEncodingState.java | 2 +-
.../erasurecode/rawcoder/DummyRawDecoder.java | 2 +-
.../erasurecode/rawcoder/DummyRawEncoder.java | 2 +-
.../rawcoder/DummyRawErasureCoderFactory.java | 2 +-
.../ozone/erasurecode/rawcoder/EncodingState.java | 2 +-
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 2 +-
.../rawcoder/RSRawErasureCoderFactory.java | 2 +-
.../rawcoder/RawErasureCoderFactory.java | 2 +-
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 2 +-
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 2 +-
.../rawcoder/XORRawErasureCoderFactory.java | 2 +-
.../ozone/erasurecode/rawcoder/package-info.java | 4 +-
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 2 +-
.../ozone/erasurecode/rawcoder/util/GF256.java | 2 +-
.../erasurecode/rawcoder/util/GaloisField.java | 2 +-
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 2 +-
.../erasurecode/rawcoder/util/package-info.java | 4 +-
hadoop-hdds/framework/pom.xml | 10 +
.../hdds/conf/DatanodeRatisServerConfig.java | 4 +-
.../hadoop/hdds/protocol/ReconfigureProtocol.java | 62 +
.../ReconfigureProtocolClientSideTranslatorPB.java | 195 ++
.../hdds/protocolPB/ReconfigureProtocolPB.java | 23 +-
.../ReconfigureProtocolServerSideTranslatorPB.java | 141 +
...inerLocationProtocolClientSideTranslatorPB.java | 11 +
.../hadoop/hdds/security/OzoneSecretKey.java | 9 +-
.../hadoop/hdds/security/OzoneSecretManager.java | 54 +-
.../hdds/security/ssl/MonitoringTimerTask.java | 16 +-
.../security/ssl/PemFileBasedKeyStoresFactory.java | 2 +-
.../hdds/security/ssl/ReloadingX509KeyManager.java | 2 +-
.../token/ContainerTokenSecretManager.java | 5 +-
.../token/OzoneBlockTokenSecretManager.java | 5 +-
.../token/ShortLivedTokenSecretManager.java | 9 +-
.../x509/certificate/authority/BaseApprover.java | 4 +-
.../authority/CAType.java} | 23 +-
.../certificate/authority/CertificateServer.java | 31 +-
.../certificate/authority/DefaultApprover.java | 2 +-
.../certificate/authority/DefaultCAServer.java | 74 +-
.../{PKIProfiles => profile}/DefaultCAProfile.java | 2 +-
.../{PKIProfiles => profile}/DefaultProfile.java | 2 +-
.../{PKIProfiles => profile}/PKIProfile.java | 2 +-
.../{PKIProfiles => profile}/package-info.java | 2 +-
.../x509/certificate/client/CertificateClient.java | 92 +-
.../client/CertificateNotification.java} | 19 +-
.../client/CommonCertificateClient.java | 14 +-
.../certificate/client/DNCertificateClient.java | 105 +-
.../client/DefaultCertificateClient.java | 716 +++-
.../certificate/client/OMCertificateClient.java | 57 -
.../certificate/client/ReconCertificateClient.java | 108 +-
.../certificate/client/SCMCertificateClient.java | 27 +-
.../utils/CertificateSignRequest.java | 4 +-
.../utils/SelfSignedCertificate.java | 4 +-
.../hadoop/hdds/security/x509/crl/CRLCodec.java | 13 +-
.../CertificateException.java | 10 +-
.../{exceptions => exception}/package-info.java | 2 +-
.../hadoop/hdds/security/x509/keys/KeyCodec.java | 12 +
.../hdds/security/x509/keys/SecurityUtil.java | 2 +-
.../org/apache/hadoop/hdds/server/OzoneAdmins.java | 13 +-
.../hadoop/hdds/server/ServiceRuntimeInfoImpl.java | 24 +-
.../hadoop/hdds/utils/BooleanTriFunction.java | 20 +-
.../hadoop/hdds/utils/DBCheckpointServlet.java | 69 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 24 +-
.../hadoop/hdds/utils/RocksDBStoreMBean.java | 160 +-
.../org/apache/hadoop/hdds/utils/db/DBProfile.java | 25 +
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 6 +
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 58 +-
.../hadoop/hdds/utils/db/RDBCheckpointManager.java | 68 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 92 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 189 +
.../org/apache/hadoop/hdds/utils/db/Table.java | 11 +
.../hadoop/hdds/utils/db/cache/CacheValue.java | 32 +-
.../db/managed/ManagedCompactRangeOptions.java} | 14 +-
.../hdds/utils/db/managed/ManagedRocksDB.java | 6 +
.../token/TestOzoneBlockTokenSecretManager.java | 6 +-
.../hdds/security/token/TokenVerifierTests.java | 2 +-
.../hdds/security/x509/CertificateClientTest.java | 74 +-
.../x509/certificate/authority/MockApprover.java | 2 +-
.../certificate/authority/TestDefaultCAServer.java | 154 +-
.../certificate/authority/TestDefaultProfile.java | 4 +-
.../client/TestDefaultCertificateClient.java | 288 +-
...tInit.java => TestDnCertificateClientInit.java} | 67 +-
.../x509/certificate/utils/TestCRLCodec.java | 13 +-
.../certificate/utils/TestCertificateCodec.java | 211 +-
.../utils}/TestCertificateSignRequest.java | 3 +-
.../utils}/TestRootCertificate.java | 14 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 16 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +
.../src/main/proto/ScmAdminProtocol.proto | 3 +
hadoop-hdds/interface-client/pom.xml | 2 +
.../src/main/proto/DatanodeClientProtocol.proto | 12 +
.../src/main/proto/ReconfigureProtocol.proto | 92 +
.../interface-client/src/main/proto/hdds.proto | 8 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 7 +
.../src/main/proto/ScmServerProtocol.proto | 2 +
hadoop-hdds/pom.xml | 7 +
hadoop-hdds/rocksdb-checkpoint-differ/README.md | 18 +
.../dev-support/findbugsExcludeFile.xml | 0
.../rocksdb-checkpoint-differ}/pom.xml | 127 +-
.../ozone/rocksdb/util/ManagedSstFileReader.java | 128 +
.../org/apache/ozone/rocksdb/util/RdbUtil.java | 61 +
.../apache/ozone/rocksdb/util}/package-info.java | 8 +-
.../org/apache/ozone/rocksdiff/CompactionNode.java | 80 +
.../apache/ozone/rocksdiff/DifferSnapshotInfo.java | 63 +
.../apache/ozone/rocksdiff/RelationshipEdge.java | 23 +-
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 1380 ++++++++
.../org/apache/ozone/rocksdiff/RocksDiffUtils.java | 54 +
.../org/apache/ozone/rocksdiff}/package-info.java | 0
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 1221 +++++++
.../src/test/resources/log4j.properties | 0
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 90 +
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 8 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 9 +
.../scm/container/CloseContainerEventHandler.java | 7 +-
.../hdds/scm/container/ContainerManagerImpl.java | 13 +-
.../hdds/scm/container/ContainerReplica.java | 12 +
.../hdds/scm/container/ContainerReportHandler.java | 6 +-
.../scm/container/ContainerStateManagerImpl.java | 33 +-
.../IncrementalContainerReportHandler.java | 15 +-
.../hdds/scm/container/balancer/MoveManager.java | 523 +++
.../AbstractOverReplicationHandler.java | 34 +-
.../replication/ContainerHealthResult.java | 52 +-
.../replication/ContainerReplicaCount.java | 7 +-
.../container/replication/ContainerReplicaOp.java | 10 +-
.../replication/ContainerReplicaPendingOps.java | 33 +-
.../replication/ECContainerReplicaCount.java | 75 +-
.../replication/ECMisReplicationHandler.java | 16 +-
.../replication/ECOverReplicationHandler.java | 132 +-
.../replication/ECUnderReplicationHandler.java | 65 +-
.../LegacyRatisContainerReplicaCount.java | 62 +
.../replication/LegacyReplicationManager.java | 832 +++--
.../replication/MisReplicationHandler.java | 56 +-
.../replication/OverReplicatedProcessor.java | 5 +-
.../replication/RatisContainerReplicaCount.java | 246 +-
.../replication/RatisMisReplicationHandler.java | 27 +-
.../replication/RatisOverReplicationHandler.java | 147 +-
.../replication/RatisUnderReplicationHandler.java | 126 +-
.../container/replication/ReplicationManager.java | 304 +-
.../replication/ReplicationManagerMetrics.java | 63 +-
.../replication/UnderReplicatedProcessor.java | 5 +-
.../replication/UnhealthyReplicationHandler.java | 5 +-
.../replication/UnhealthyReplicationProcessor.java | 32 +-
.../health/ClosedWithUnhealthyReplicasHandler.java | 2 +-
.../health/DeletingContainerHandler.java | 3 +-
.../health/ECReplicationCheckHandler.java | 32 +-
.../replication/health/EmptyContainerHandler.java | 2 +-
...Handler.java => MismatchedReplicasHandler.java} | 68 +-
.../health/RatisReplicationCheckHandler.java | 85 +-
.../RatisUnhealthyReplicationCheckHandler.java | 196 ++
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 62 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 87 +-
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 27 +-
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 2 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 8 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 17 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 5 +-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 12 +
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 18 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 29 +-
.../hdds/scm/pipeline/ECPipelineProvider.java | 25 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 4 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 23 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 17 +
.../hdds/scm/server/SCMBlockProtocolServer.java | 14 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 106 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 17 -
.../hdds/scm/server/SCMSecurityProtocolServer.java | 11 +-
.../hdds/scm/server/StorageContainerManager.java | 116 +-
.../scm/server/StorageContainerManagerStarter.java | 2 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 166 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 28 +-
.../container/TestCloseContainerEventHandler.java | 1 +
.../hdds/scm/container/TestContainerReplica.java | 53 +
.../scm/container/TestContainerReportHandler.java | 5 +
.../scm/container/TestContainerStateManager.java | 5 +
.../TestIncrementalContainerReportHandler.java | 29 +-
.../scm/container/balancer/TestMoveManager.java | 514 +++
.../algorithms/TestContainerPlacementFactory.java | 7 +
.../container/replication/ReplicationTestUtil.java | 38 +
.../TestContainerReplicaPendingOps.java | 116 +-
.../replication/TestECContainerReplicaCount.java | 77 +-
.../replication/TestECMisReplicationHandler.java | 3 +-
.../replication/TestECOverReplicationHandler.java | 130 +-
.../replication/TestECUnderReplicationHandler.java | 106 +-
.../replication/TestLegacyReplicationManager.java | 3653 +++++++++++---------
.../replication/TestMisReplicationHandler.java | 14 +-
.../replication/TestOverReplicatedProcessor.java | 41 +-
.../TestRatisContainerReplicaCount.java | 235 ++
.../TestRatisMisReplicationHandler.java | 15 +-
.../TestRatisOverReplicationHandler.java | 81 +-
.../TestRatisUnderReplicationHandler.java | 57 +-
.../replication/TestReplicationManager.java | 233 +-
.../replication/TestUnderReplicatedProcessor.java | 55 +-
.../TestClosedWithUnhealthyReplicasHandler.java | 2 +-
.../health/TestDeletingContainerHandler.java | 2 +-
.../health/TestECReplicationCheckHandler.java | 120 +
.../health/TestEmptyContainerHandler.java | 2 +-
...ler.java => TestMismatchedReplicasHandler.java} | 94 +-
.../health/TestRatisReplicationCheckHandler.java | 155 +-
.../TestRatisUnhealthyReplicationCheckHandler.java | 334 ++
.../scm/node/DatanodeAdminMonitorTestUtil.java | 78 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 60 +
.../hadoop/hdds/scm/node/TestNodeStatus.java | 68 +
.../hdds/scm/pipeline/TestECPipelineProvider.java | 76 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 24 +-
.../java/org/apache/ozone/test/SpyInputStream.java | 32 +-
.../org/apache/ozone/test/SpyOutputStream.java | 26 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 12 +-
.../hdds/scm/cli/ContainerOperationClient.java | 5 +
.../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 5 +
.../hadoop/hdds/scm/cli/cert/InfoSubcommand.java | 2 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 12 +-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 4 +
.../apache/hadoop/ozone/client/ObjectStore.java | 43 +
.../apache/hadoop/ozone/client/OzoneBucket.java | 344 +-
.../apache/hadoop/ozone/client/OzoneSnapshot.java | 145 +
.../apache/hadoop/ozone/client/OzoneVolume.java | 232 +-
.../client/checksum/ECFileChecksumHelper.java | 23 +-
.../checksum/ReplicatedFileChecksumHelper.java | 4 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 13 +
.../client/io/BlockOutputStreamEntryPool.java | 23 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 23 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 57 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 23 +-
.../hadoop/ozone/client/io/OzoneOutputStream.java | 25 +-
.../ozone/client/protocol/ClientProtocol.java | 47 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 248 +-
.../hadoop/ozone/client/MockOmTransport.java | 22 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 11 +-
hadoop-ozone/common/pom.xml | 2 -
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 38 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 121 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 8 +-
.../apache/hadoop/ozone/om/IOmMetadataReader.java | 123 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 25 +-
.../java/org/apache/hadoop/ozone/om/S3Batcher.java | 46 +
.../org/apache/hadoop/ozone/om/S3SecretCache.java | 53 +-
.../apache/hadoop/ozone/om/S3SecretFunction.java | 42 +-
.../hadoop/ozone/om/S3SecretLockedManager.java | 100 +
.../apache/hadoop/ozone/om/S3SecretManager.java | 81 +-
.../org/apache/hadoop/ozone/om/S3SecretStore.java | 56 +
.../hadoop/ozone/om/exceptions/OMException.java | 6 +-
.../ozone/om/exceptions/OMNotLeaderException.java | 45 +-
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 15 +-
.../om/ha/HadoopRpcOMFailoverProxyProvider.java | 12 +-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 26 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 4 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 18 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 3 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 19 +
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 515 +++
.../hadoop/ozone/om/lock/IOzoneManagerLock.java | 98 +
.../hadoop/ozone/om/lock/OmReadOnlyLock.java | 167 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 36 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 12 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 130 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 26 +-
.../ozone/om/protocolPB/OMAdminProtocolPB.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 162 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 -
.../ozone/security/OzoneTokenIdentifier.java | 3 +-
.../hadoop/ozone/snapshot/SnapshotDiffReport.java | 233 ++
.../hadoop/ozone/snapshot/package-info.java} | 17 +-
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 2 +
.../main/resources/ozone-version-info.properties | 1 +
.../hadoop/ozone/om/helpers/TestOmBucketArgs.java | 2 +-
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 11 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 6 +-
.../ozone/om/helpers/TestOmSnapshotInfo.java | 147 +
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 9 +-
hadoop-ozone/csi/pom.xml | 3 -
.../org/apache/hadoop/ozone/csi/CsiServer.java | 6 +-
.../dev-support/intellij/ozone-site-ha.xml | 174 +
...torageContainerManager.xml => Datanode1-ha.xml} | 8 +-
...torageContainerManager.xml => Datanode2-ha.xml} | 8 +-
...torageContainerManager.xml => Datanode3-ha.xml} | 8 +-
...ageContainerManager.xml => OzoneFsShell-ha.xml} | 13 +-
...ageContainerManager.xml => OzoneManager-ha.xml} | 8 +-
...ontainerManager.xml => OzoneManagerInit-ha.xml} | 8 +-
...orageContainerManager.xml => OzoneShell-ha.xml} | 8 +-
...geContainerManager.xml => PrimordialSCM-ha.xml} | 4 +-
...ntainerManager.xml => PrimordialSCMInit-ha.xml} | 4 +-
.../{StorageContainerManager.xml => Recon-ha.xml} | 8 +-
.../{StorageContainerManager.xml => Scm2-ha.xml} | 4 +-
...geContainerManager.xml => Scm2Bootstrap-ha.xml} | 4 +-
.../{StorageContainerManager.xml => Scm3-ha.xml} | 4 +-
...geContainerManager.xml => Scm3Bootstrap-ha.xml} | 4 +-
.../{StorageContainerManager.xml => ScmRoles.xml} | 8 +-
.../runConfigurations/StorageContainerManager.xml | 2 +-
.../src/main/compose/ozone-ha/docker-compose.yaml | 4 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 9 +-
.../dist/src/main/compose/ozones3-haproxy/test.sh | 4 +-
.../src/main/compose/ozonesecure/docker-config | 7 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 22 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 19 +-
.../dist/src/main/smoketest/basic/links.robot | 16 +-
.../main/smoketest/basic/ozone-shell-single.robot | 2 +-
.../src/main/smoketest/basic/ozone-shell.robot | 2 +-
.../dist/src/main/smoketest/createbucketenv.robot | 2 +-
.../dist/src/main/smoketest/createmrenv.robot | 2 +-
.../main/smoketest/debug/ozone-debug-tests.robot | 2 +-
.../src/main/smoketest/ec/backward-compat.robot | 106 +-
.../dist/src/main/smoketest/ec/basic.robot | 89 +-
.../dist/src/main/smoketest/ec/lib.resource | 18 +-
.../ozone-shell-single.robot => ec/read.robot} | 27 +-
.../dist/src/main/smoketest/gdpr/gdpr.robot | 8 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 46 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 6 +-
.../dist/src/main/smoketest/ozonefs/setup.robot | 12 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 8 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 1 +
.../dist/src/main/smoketest/s3/webui.robot | 2 +-
.../main/smoketest/security/ozone-secure-fs.robot | 51 +
.../smoketest/security/ozone-secure-tenant.robot | 7 +
.../src/main/smoketest/snapshot/snapshot-fs.robot | 60 +
.../main/smoketest/snapshot/snapshot-setup.robot | 63 +
.../src/main/smoketest/snapshot/snapshot-sh.robot | 57 +
.../dist/src/shell/conf/dn-audit-log4j2.properties | 18 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 18 +-
.../src/shell/conf/s3g-audit-log4j2.properties | 18 +-
.../src/shell/conf/scm-audit-log4j2.properties | 18 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 1 -
hadoop-ozone/integration-test/pom.xml | 5 -
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 255 ++
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 3 +-
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 2 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 8 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 103 +
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 55 +-
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 350 ++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 201 +-
.../ozone/contract/ITestOzoneContractCreate.java | 23 +-
.../ozone/contract/ITestOzoneContractDelete.java | 23 +-
.../contract/ITestOzoneContractGetFileStatus.java | 28 +-
.../fs/ozone/contract/ITestOzoneContractMkdir.java | 23 +-
.../fs/ozone/contract/ITestOzoneContractOpen.java | 23 +-
.../ozone/contract/ITestOzoneContractRename.java | 23 +-
.../ozone/contract/ITestOzoneContractRootDir.java | 23 +-
.../fs/ozone/contract/ITestOzoneContractUtils.java | 61 -
.../hadoop/fs/ozone/contract/OzoneContract.java | 39 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 36 +-
.../hdds/scm/TestSCMDatanodeProtocolServer.java | 4 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 2 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 3 +
.../container/TestScmApplyTransactionFailure.java | 147 +
.../TestRatisPipelineCreateAndDestroy.java | 3 +
.../hdds/scm/storage/TestContainerCommandsEC.java | 432 +--
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 4 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 +-
.../hadoop/ozone/TestContainerOperations.java | 34 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 638 +++-
.../ozone/TestStorageContainerManagerHelper.java | 2 +-
.../ozone/client/CertificateClientTestImpl.java | 154 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 26 +-
.../rpc/TestBlockOutputStreamFlushDelay.java | 26 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 2 +
...estBlockOutputStreamWithFailuresFlushDelay.java | 2 +
.../ozone/client/rpc/TestECKeyOutputStream.java | 5 +-
.../client/rpc/TestFailureHandlingByClient.java | 6 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 2 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 291 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 3 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 80 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 2 +-
.../client/rpc/read/TestChunkInputStream.java | 109 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 +
.../ozone/container/TestContainerReplication.java | 50 +-
.../ozone/container/TestECContainerRecovery.java | 40 +-
.../apache/hadoop/ozone/container/TestHelper.java | 20 +-
.../commandhandler/TestBlockDeletion.java | 15 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 55 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 4 +-
.../replication/TestContainerReplication.java | 173 +
.../ozone/container/replication/package-info.java | 33 +-
.../container/server/TestContainerServer.java | 3 +-
.../server/TestSecureContainerServer.java | 6 +-
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 269 ++
.../hadoop/ozone/fsck/TestContainerMapper.java | 8 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 1 +
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 418 ++-
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 58 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 24 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 4 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 57 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 9 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 65 +
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 835 +++++
.../hadoop/ozone/om/TestOmSnapshotFileSystem.java | 658 ++++
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 6 +
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 49 +
.../ozone/om/TestOzoneManagerHAWithData.java | 129 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 1 +
.../ozone/om/TestOzoneManagerListVolumes.java | 26 +-
.../hadoop/ozone/om/TestOzoneManagerRestart.java | 4 +
.../hadoop/ozone/om/TestSecureOzoneManager.java | 18 +-
.../om/snapshot/TestOzoneSnapshotRestore.java | 333 ++
.../ozone/recon/TestReconWithOzoneManager.java | 57 -
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 14 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 8 +-
.../hadoop/ozone/reconfig/TestOmReconfigure.java | 100 +
.../hadoop/ozone/reconfig/TestScmReconfigure.java | 101 +
.../scm/node/TestDecommissionAndMaintenance.java | 2 -
.../hadoop/ozone/shell/TestOzoneShellHA.java | 58 +-
.../ozone/shell/TestTransferLeadershipShell.java | 137 +
.../src/test/resources/contract/ozone.xml | 10 +
hadoop-ozone/interface-client/pom.xml | 2 -
.../src/main/proto/OmClientProtocol.proto | 120 +
.../apache/hadoop/ozone/om/ExpiredOpenKeys.java | 68 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 36 +-
.../ozone/om/codec/OmDBDiffReportEntryCodec.java | 53 +
.../ozone/om/codec/OmDBSnapshotInfoCodec.java | 58 +
hadoop-ozone/ozone-manager/pom.xml | 24 +
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 111 -
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 24 +-
.../java/org/apache/hadoop/ozone/om/IOzoneAcl.java | 31 -
.../org/apache/hadoop/ozone/om/KeyManager.java | 11 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 365 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 116 +-
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 184 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 316 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 583 ++++
.../hadoop/ozone/om/OmMetadataReaderMetrics.java | 61 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 265 ++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 318 ++
.../apache/hadoop/ozone/om/OmSnapshotMetrics.java | 163 +
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 16 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 39 +-
.../hadoop/ozone/om/OzoneListStatusHelper.java | 49 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 1097 +++---
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 9 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 2 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 104 -
.../hadoop/ozone/om/S3SecretManagerImpl.java | 93 +-
.../apache/hadoop/ozone/om/SnapshotChainInfo.java | 57 +
.../hadoop/ozone/om/SnapshotChainManager.java | 512 +++
.../hadoop/ozone/om/SstFilteringService.java | 212 ++
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 22 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 6 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 111 -
.../hadoop/ozone/om/codec/OMDBDefinition.java | 13 +-
.../org/apache/hadoop/ozone/om/ha/OMHAMetrics.java | 132 +
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 14 +-
.../ozone/om/lock/RegularBucketLockStrategy.java | 20 +-
.../ozone/om/multitenant/AuthorizerLock.java | 15 +-
.../ozone/om/multitenant/AuthorizerLockImpl.java | 24 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 443 ++-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 55 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 10 +-
.../BucketLayoutAwareOMKeyRequestFactory.java | 2 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 17 +-
.../om/request/bucket/OMBucketCreateRequest.java | 123 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 56 +
.../request/bucket/OMBucketSetPropertyRequest.java | 12 +-
.../om/request/file/OMDirectoryCreateRequest.java | 14 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 3 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 13 +-
.../request/file/OMFileCreateRequestWithFSO.java | 7 +-
.../ozone/om/request/file/OMFileRequest.java | 113 +-
.../om/request/key/OMAllocateBlockRequest.java | 3 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 3 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 84 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 54 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 13 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 6 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 88 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 6 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 8 +-
.../S3MultipartUploadCommitPartRequest.java | 3 +-
.../S3MultipartUploadCompleteRequest.java | 5 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 5 +-
.../om/request/s3/security/OMSetSecretRequest.java | 85 +-
.../om/request/s3/security/S3GetSecretRequest.java | 110 +-
.../request/s3/security/S3RevokeSecretRequest.java | 45 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 39 +-
.../tenant/OMTenantRevokeUserAccessIdRequest.java | 20 +-
.../request/snapshot/OMSnapshotCreateRequest.java | 190 +
.../request/snapshot/OMSnapshotDeleteRequest.java | 227 ++
.../ozone/om/request/snapshot}/package-info.java | 4 +-
.../om/request/validation/RequestValidations.java | 25 +-
.../om/request/validation/ValidatorRegistry.java | 7 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 16 +
.../ozone/om/response/key/OMKeyCommitResponse.java | 16 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 14 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 45 +-
.../om/response/key/OMTrashRecoverResponse.java | 15 +-
.../S3InitiateMultipartUploadResponseWithFSO.java | 3 +-
.../S3MultipartUploadCompleteResponse.java | 15 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 +-
.../response/s3/security/OMSetSecretResponse.java | 12 +-
.../response/s3/security/S3GetSecretResponse.java | 20 +-
.../s3/security/S3RevokeSecretResponse.java | 14 +-
.../tenant/OMTenantAssignUserAccessIdResponse.java | 21 +-
.../tenant/OMTenantRevokeUserAccessIdResponse.java | 13 +-
.../OMSnapshotCreateResponse.java} | 65 +-
.../OMSnapshotDeleteResponse.java} | 52 +-
.../ozone/om/response/snapshot}/package-info.java | 4 +-
.../ozone/om/service/OMRangerBGSyncService.java | 49 +-
.../ozone/om/service/OpenKeyCleanupService.java | 67 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 2 +-
.../hadoop/ozone/om/snapshot/PersistentList.java} | 22 +-
.../hadoop/ozone/om/snapshot/PersistentMap.java | 21 +-
.../hadoop/ozone/om/snapshot/PersistentSet.java | 23 +-
.../ozone/om/snapshot/RocksDbPersistentList.java | 141 +
.../ozone/om/snapshot/RocksDbPersistentMap.java | 72 +
.../ozone/om/snapshot/RocksDbPersistentSet.java | 87 +
.../ozone/om/snapshot/SnapshotDiffManager.java | 556 +++
...OzoneManagerProtocolServerSideTranslatorPB.java | 104 +-
.../protocolPB/OzoneManagerRequestHandler.java | 96 +-
.../hadoop/ozone/security/OMCertificateClient.java | 205 ++
.../OzoneDelegationTokenSecretManager.java | 30 +-
.../ozone/security/acl/OzoneNativeAuthorizer.java | 10 +-
.../hadoop/ozone/om/TestAuthorizerLockImpl.java | 37 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 2 +-
.../org/apache/hadoop/ozone/om/TestOMStorage.java | 315 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 241 +-
.../hadoop/ozone/om/TestOmSnapshotManager.java | 128 +
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 14 +-
.../apache/hadoop/ozone/om/TestSnapshotChain.java | 306 ++
.../apache/hadoop/ozone/om/TestSnapshotInfo.java | 102 +
.../hadoop/ozone/om/TestSstFilteringService.java | 294 ++
.../hadoop/ozone/om/failover/TestOMFailovers.java | 6 +-
.../apache/hadoop/ozone/om/ha/TestOMHAMetrics.java | 61 +
.../om/ratis/TestOzoneManagerDoubleBuffer.java | 218 ++
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../ozone/om/request/OMRequestTestUtils.java | 86 +
.../ozone/om/request/bucket/TestBucketRequest.java | 4 +
.../request/bucket/TestOMBucketCreateRequest.java | 26 +-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 4 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 13 +-
.../om/request/key/TestOMKeyCommitRequest.java | 40 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 11 +-
.../om/request/key/TestOMKeyRenameRequest.java | 121 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 100 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 4 +
.../s3/security/TestS3GetSecretRequest.java | 13 +-
.../snapshot/TestOMSnapshotCreateRequest.java | 293 ++
.../snapshot/TestOMSnapshotDeleteRequest.java | 331 ++
.../ozone/om/request/snapshot}/package-info.java | 4 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 23 +
.../om/response/key/TestOMKeyCommitResponse.java | 9 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 5 +-
.../om/response/key/TestOMKeyRenameResponse.java | 186 +-
.../key/TestOMKeyRenameResponseWithFSO.java | 94 +
.../s3/multipart/TestS3MultipartResponse.java | 2 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 85 +
.../snapshot/TestOMSnapshotCreateResponse.java | 122 +
.../snapshot/TestOMSnapshotDeleteResponse.java | 125 +
.../ozone/om/response/snapshot}/package-info.java | 4 +-
.../om/service/TestOpenKeyCleanupService.java | 60 +-
.../om/snapshot/TestRocksDbPersistentList.java | 118 +
.../om/snapshot/TestRocksDbPersistentMap.java | 127 +
.../om/snapshot/TestRocksDbPersistentSet.java | 143 +
.../ozone/om/upgrade/TestOMVersionManager.java | 4 +-
.../hadoop/ozone/security/S3SecretStoreMap.java | 57 +
.../security/TestOmCertificateClientInit.java | 65 +-
.../TestOzoneDelegationTokenSecretManager.java | 113 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 6 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 12 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 21 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 179 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 55 +-
.../hadoop/fs/ozone/CapableOzoneFSInputStream.java | 4 +-
.../fs/ozone/CapableOzoneFSOutputStream.java | 61 +
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 2 +
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 26 +-
.../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 14 +-
.../hadoop/fs/ozone/OzoneStreamCapabilities.java | 47 -
.../org/apache/hadoop/fs/ozone/TestOFSPath.java | 26 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 3 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 20 +
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 7 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 7 +
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 7 +
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 7 +
hadoop-ozone/pom.xml | 11 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 109 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 24 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 43 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 6 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../recon/api/handlers/BucketEntityHandler.java | 36 +-
.../ozone/recon/api/handlers/BucketHandler.java | 4 +
.../recon/api/handlers/DirectoryEntityHandler.java | 26 +-
.../ozone/recon/api/handlers/FSOBucketHandler.java | 20 +
.../ozone/recon/api/handlers/KeyEntityHandler.java | 23 +-
.../recon/api/handlers/LegacyBucketHandler.java | 14 +
.../recon/api/handlers/RootEntityHandler.java | 30 +-
.../recon/api/handlers/UnknownEntityHandler.java | 8 +-
.../recon/api/handlers/VolumeEntityHandler.java | 32 +-
.../ozone/recon/api/types/BucketObjectDBInfo.java | 168 +
...mespaceSummaryResponse.java => CountStats.java} | 70 +-
.../ozone/recon/api/types/KeyObjectDBInfo.java | 154 +
.../recon/api/types/NamespaceSummaryResponse.java | 138 +-
.../hadoop/ozone/recon/api/types/ObjectDBInfo.java | 140 +
.../ozone/recon/api/types/PipelineMetadata.java | 9 +-
.../ozone/recon/api/types/VolumeObjectDBInfo.java | 78 +
.../ozone/recon/fsck/ReconSafeModeMgrTask.java | 122 +
.../recon/recovery/ReconOmMetadataManagerImpl.java | 2 +-
.../ozone/recon/scm/ReconSafeModeManager.java | 12 +-
.../scm/ReconStorageContainerManagerFacade.java | 19 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 2 +-
.../impl/StorageContainerServiceProviderImpl.java | 5 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 114 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 14 +-
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 15 +-
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 17 +
.../webapps/recon/ozone-recon-web/api/db.json | 345 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 2 +-
.../src/views/diskUsage/diskUsage.tsx | 131 +-
.../src/views/pipelines/pipelines.less | 9 +-
.../src/views/pipelines/pipelines.tsx | 10 +-
.../ozone/recon/api/TestContainerEndpoint.java | 68 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 44 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 61 +-
.../recon/api/TestNSSummaryEndpointWithLegacy.java | 62 +-
.../hadoop/ozone/recon/common/CommonUtils.java | 225 ++
.../ozone/recon/fsck/TestContainerHealthTask.java | 6 +
.../ozone/recon/tasks/TestFileSizeCountTask.java | 222 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 12 +-
hadoop-ozone/s3gateway/pom.xml | 5 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 2 +-
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 9 +-
.../org/apache/hadoop/ozone/s3/TracingFilter.java | 29 +-
.../hadoop/ozone/s3/commontypes/CommonPrefix.java | 8 +-
.../{CommonPrefix.java => EncodingTypeObject.java} | 40 +-
.../hadoop/ozone/s3/commontypes/KeyMetadata.java | 6 +-
.../ozone/s3/commontypes/ObjectKeyNameAdapter.java | 14 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 32 +-
.../ozone/s3/endpoint/ListObjectResponse.java | 33 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 18 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 3 +
.../hadoop/ozone/client/ClientProtocolStub.java | 28 +
.../hadoop/ozone/client/ObjectStoreStub.java | 17 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 82 +-
.../hadoop/ozone/client/OzoneOutputStreamStub.java | 2 +-
.../hadoop/ozone/client/OzoneVolumeStub.java | 86 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 23 +-
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 121 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 96 +-
hadoop-ozone/tools/pom.xml | 10 +-
.../ozone/admin/nssummary/NSSummaryAdmin.java | 9 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 2 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 3 +-
.../ozone/admin/om/TransferOmLeaderSubCommand.java | 73 +
.../ReconfigureCommands.java} | 52 +-
.../reconfig/ReconfigurePropertiesSubcommand.java | 54 +
.../admin/reconfig/ReconfigureStartSubcommand.java | 50 +
.../reconfig/ReconfigureStatusSubcommand.java | 90 +
.../admin/reconfig/ReconfigureSubCommandUtil.java | 46 +
.../hadoop/ozone/admin/reconfig}/package-info.java | 15 +-
.../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 5 +-
.../admin/scm/TransferScmLeaderSubCommand.java | 72 +
.../org/apache/hadoop/ozone/debug/DBScanner.java | 2 +-
.../org/apache/hadoop/ozone/debug/DropTable.java | 2 +-
.../ozone/debug/container/ExportSubcommand.java | 6 +-
.../ozone/freon/ClosedContainerReplicator.java | 21 +-
.../ozone/freon/DatanodeSimulationState.java | 375 ++
.../hadoop/ozone/freon/DatanodeSimulator.java | 587 ++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 2 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 36 +
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 39 +-
.../shell/snapshot/BucketSnapshotHandler.java} | 19 +-
.../shell/snapshot/CreateSnapshotHandler.java | 62 +
.../shell/snapshot/DeleteSnapshotHandler.java | 61 +
.../ozone/shell/snapshot/ListSnapshotHandler.java | 62 +
.../ozone/shell/snapshot/SnapshotCommands.java | 75 +
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 67 +
.../hadoop/ozone/shell/snapshot/SnapshotUri.java | 42 +-
.../hadoop/ozone/shell/snapshot/package-info.java} | 24 +-
.../hadoop/ozone/shell/TestOzoneAddress.java | 105 +-
pom.xml | 132 +-
830 files changed, 44094 insertions(+), 11555 deletions(-)
copy
hadoop-hdds/{container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerDownloader.java
=>
common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/DuplicatedPipelineIdException.java}
(57%)
copy
hadoop-hdds/{container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerDownloader.java
=>
common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/InvalidPipelineStateException.java}
(57%)
copy
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/{impl/RandomContainerDeletionChoosingPolicy.java
=> interfaces/ContainerDeletionChoosingPolicyTemplate.java} (57%)
delete mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionSupervisor.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/AbstractReplicationTask.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerImporter.java
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSafeModeManager.java
=>
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerUploader.java
(62%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/CopyContainerResponseStream.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcContainerUploader.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/PushReplicator.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerOutputStream.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/SendContainerRequestHandler.java
delete mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionSupervisor.java
rename
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/{TestGrpcOutputStream.java
=> GrpcOutputStreamTest.java} (87%)
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestCopyContainerCompression.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestCopyContainerResponseStream.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestGrpcReplicationService.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestPushReplicator.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestSendContainerOutputStream.java
create mode 100644 hadoop-hdds/docs/content/feature/ErasureCoding.zh.md
create mode 100644
hadoop-hdds/docs/content/feature/GrafanaOzoneObjectMetrics.png
create mode 100644 hadoop-hdds/docs/content/feature/GrafanaOzoneOverall.png
create mode 100644 hadoop-hdds/docs/content/feature/GrafanaOzoneRPCMetrics.png
create mode 100644 hadoop-hdds/docs/content/feature/Reconfigurability.md
create mode 100644 hadoop-hdds/docs/content/feature/Reconfigurability.zh.md
create mode 100644 hadoop-hdds/docs/content/tools/LogsInOzone.md
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/ReconfigureProtocol.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/ReconfigureProtocolClientSideTranslatorPB.java
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolPB.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/ReconfigureProtocolPB.java
(60%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/ReconfigureProtocolServerSideTranslatorPB.java
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/{certificates/utils/package-info.java
=> certificate/authority/CAType.java} (62%)
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/{PKIProfiles
=> profile}/DefaultCAProfile.java (99%)
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/{PKIProfiles
=> profile}/DefaultProfile.java (99%)
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/{PKIProfiles
=> profile}/PKIProfile.java (99%)
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/{PKIProfiles
=> profile}/package-info.java (98%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/{certificates/utils/package-info.java
=> certificate/client/CertificateNotification.java} (59%)
delete mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/{certificates
=> certificate}/utils/CertificateSignRequest.java (98%)
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/{certificates
=> certificate}/utils/SelfSignedCertificate.java (98%)
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/{exceptions
=> exception}/CertificateException.java (92%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/{exceptions
=> exception}/package-info.java (93%)
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSafeModeManager.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/BooleanTriFunction.java
(63%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/{security/x509/exceptions/package-info.java
=> utils/db/managed/ManagedCompactRangeOptions.java} (71%)
copy
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/{TestCertificateClientInit.java
=> TestDnCertificateClientInit.java} (71%)
rename
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/{certificates
=> certificate/utils}/TestCertificateSignRequest.java (98%)
rename
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/{certificates
=> certificate/utils}/TestRootCertificate.java (95%)
create mode 100644
hadoop-hdds/interface-client/src/main/proto/ReconfigureProtocol.proto
create mode 100644 hadoop-hdds/rocksdb-checkpoint-differ/README.md
copy {hadoop-ozone/insight =>
hadoop-hdds/rocksdb-checkpoint-differ}/dev-support/findbugsExcludeFile.xml
(100%)
copy {hadoop-ozone/common => hadoop-hdds/rocksdb-checkpoint-differ}/pom.xml
(58%)
create mode 100644
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/ManagedSstFileReader.java
create mode 100644
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java
copy
hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils
=>
rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util}/package-info.java
(83%)
create mode 100644
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java
create mode 100644
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/DifferSnapshotInfo.java
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSafeModeManager.java
=>
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RelationshipEdge.java
(72%)
create mode 100644
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
create mode 100644
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
copy hadoop-hdds/{container-service/src/main/java/org/apache/hadoop/ozone =>
rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff}/package-info.java
(100%)
create mode 100644
hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
copy {hadoop-ozone/common =>
hadoop-hdds/rocksdb-checkpoint-differ}/src/test/resources/log4j.properties
(100%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/MoveManager.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/LegacyRatisContainerReplicaCount.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/{ClosedWithMismatchedReplicasHandler.java
=> MismatchedReplicasHandler.java} (50%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisUnhealthyReplicationCheckHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReplica.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestMoveManager.java
rename
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/{TestClosedWithMismatchedReplicasHandler.java
=> TestMismatchedReplicasHandler.java} (70%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisUnhealthyReplicationCheckHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStatus.java
rename
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMEchoRPCRequest.java
=>
hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/SpyInputStream.java
(55%)
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSafeModeManager.java
=>
hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/SpyOutputStream.java
(56%)
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneSnapshot.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3Batcher.java
copy
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
=>
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretCache.java
(57%)
copy
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
=>
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretFunction.java
(57%)
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretLockedManager.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/S3SecretStore.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/IOzoneManagerLock.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OmReadOnlyLock.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
copy
hadoop-ozone/{recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/pipelines/pipelines.less
=> common/src/main/java/org/apache/hadoop/ozone/snapshot/package-info.java}
(82%)
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
create mode 100644 hadoop-ozone/dev-support/intellij/ozone-site-ha.xml
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Datanode1-ha.xml} (78%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Datanode2-ha.xml} (69%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Datanode3-ha.xml} (69%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> OzoneFsShell-ha.xml} (70%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> OzoneManager-ha.xml} (80%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> OzoneManagerInit-ha.xml} (80%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> OzoneShell-ha.xml} (80%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> PrimordialSCM-ha.xml} (85%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> PrimordialSCMInit-ha.xml} (83%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Recon-ha.xml} (80%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Scm2-ha.xml} (84%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Scm2Bootstrap-ha.xml} (83%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Scm3-ha.xml} (84%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Scm3Bootstrap-ha.xml} (83%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> ScmRoles.xml} (84%)
copy hadoop-ozone/dist/src/main/smoketest/{basic/ozone-shell-single.robot =>
ec/read.robot} (57%)
create mode 100644
hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-fs.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-setup.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/snapshot/snapshot-sh.robot
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java
delete mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractUtils.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestScmApplyTransactionFailure.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/replication/TestContainerReplication.java
copy
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerDownloader.java
=>
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
(53%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotFileSystem.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneSnapshotRestore.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfigure.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestScmReconfigure.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestTransferLeadershipShell.java
create mode 100644
hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/ExpiredOpenKeys.java
create mode 100644
hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmDBDiffReportEntryCodec.java
create mode 100644
hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/codec/OmDBSnapshotInfoCodec.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
copy
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/S3GAction.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReaderMetrics.java
(57%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotMetrics.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainInfo.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ha/OMHAMetrics.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotDeleteRequest.java
copy
{hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot}/package-info.java
(88%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{s3/security/OMSetSecretResponse.java
=> snapshot/OMSnapshotCreateResponse.java} (52%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{s3/security/S3RevokeSecretResponse.java
=> snapshot/OMSnapshotDeleteResponse.java} (55%)
copy
{hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot}/package-info.java
(87%)
copy
hadoop-ozone/{recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSafeModeManager.java
=>
ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentList.java}
(72%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentMap.java
(75%)
rename
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/package-info.java
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/PersistentSet.java
(73%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentList.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentMap.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/RocksDbPersistentSet.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OMCertificateClient.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotChain.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSnapshotInfo.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMHAMetrics.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBuffer.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotCreateRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotDeleteRequest.java
copy
{hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions
=>
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot}/package-info.java
(88%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyRenameResponseWithFSO.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java
rename
{hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions
=>
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot}/package-info.java
(88%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRocksDbPersistentList.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRocksDbPersistentMap.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestRocksDbPersistentSet.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/S3SecretStoreMap.java
rename
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestCertificateClientInit.java
=>
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOmCertificateClientInit.java
(73%)
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/CapableOzoneFSOutputStream.java
delete mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneStreamCapabilities.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/BucketObjectDBInfo.java
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/{NamespaceSummaryResponse.java
=> CountStats.java} (59%)
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyObjectDBInfo.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ObjectDBInfo.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/VolumeObjectDBInfo.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconSafeModeMgrTask.java
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/common/CommonUtils.java
copy
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/{CommonPrefix.java
=> EncodingTypeObject.java} (51%)
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/TransferOmLeaderSubCommand.java
copy
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/{scm/ScmAdmin.java
=> reconfig/ReconfigureCommands.java} (50%)
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/ReconfigurePropertiesSubcommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/ReconfigureStartSubcommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/ReconfigureStatusSubcommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig/ReconfigureSubCommandUtil.java
rename
{hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificates
=>
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/reconfig}/package-info.java
(81%)
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/scm/TransferScmLeaderSubCommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulationState.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java
copy
hadoop-ozone/{recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSafeModeManager.java
=>
tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/BucketSnapshotHandler.java}
(66%)
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/CreateSnapshotHandler.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/DeleteSnapshotHandler.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotCommands.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java
copy
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawErasureCoderFactory.java
=>
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotUri.java
(50%)
copy
hadoop-ozone/{s3gateway/src/test/java/org/apache/hadoop/ozone/s3/commontypes/TestObjectKeyNameAdapter.java
=>
tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/package-info.java}
(56%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
