This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 0bcd697107 HDDS-7690. [Snapshot] Use SST file list output from
compaction DAG as SnapshotDiff input (#4119)
add 0edb2ecfe4 HDDS-7217. OM logs wrong bucket layout (#3750)
add 2737d3d065 HDDS-7208. Erasure coding and encryption are not flagged on
FileStatus (#3768)
add e0ea7df9e5 HDDS-7259. Fix uncounted blocksDeleted in
BlockDeletingService (#3779)
add 099422c5db HDDS-7293. Bump jackson2 to 2.13.4 (#3808)
add 625e84f95e HDDS-7286. Clean up ContainerManager (#3797)
add e7f4c05d09 HDDS-7294. Move ozone manager background services to a
separate package (#3805)
add 69d4dbd95a HDDS-7302. Fix inaccurate container block count caused by
missing block (#3814)
add 17753bc93c HDDS-7287. Send deleteBlocksRequest with correct retry
count (#3799)
add b808deb8c8 HDDS-6664. Implements getUri in TrashOzoneFileSystem (#3795)
add 32df39e8c8 HDDS-7299. Migrate simple tests in hdds-container-service
to JUnit5 (#3813)
add 834ef98018 HDDS-7230. Implement GetKeyInfo API (#3780)
add f0040c294b HDDS-7304. EC: EC Decode can fail when byteBuffer from
elastic pool is larger than chunksize (#3817)
add afb47176b7 HDDS-7271. Ozone Integration test shows memory leak
(graceful shutdown cleanup) (#3787)
add bb46766e23 Revert "HDDS-7271. Ozone Integration test shows memory leak
(graceful shutdown cleanup) (#3787)"
add 74deb5959a HDDS-7236. Enable Recon SCM DB bootstrap by default. (#3823)
add 0a5fc405e5 HDDS-6440. Handle custom metadata (x-amz-meta) during
put-object through S3 API. (#3728)
add e529ed4586 HDDS-7088. OM incorrectly detects SCM Ratis Group ID when
OM and SCM are colocated with same Ratis storage directory. (#3809)
add 54af0dacb0 HDDS-7313. Update github actions for Node16 (#3825)
add 15217fe59a HDDS-7271. Ozone Integration test shows memory leak
(graceful shutdown cleanup) (#3826)
add a905b4bbb6 HDDS-6946. Bump Hadoop to 3.3.4 (#3557)
add 2bfb6d9ec5 HDDS-7315 SCM order of close of dbstore and other services
(#3830)
add b18fb595e4 HDDS-7308 Removed cluster2 from serviceIds (#3819)
add 4515fe20c2 HDDS-7291. Fixing exception handling in case of non
positive replica index (#3806)
add 58f0749b52 HDDS-7270. Fix bug in checking healthy replica placement
policy check in EC underReplication handler (#3828)
add 6e2de99324 HDDS-7308. Fix config example syntax highlight in OM HA
(#3833)
add 330b2e308d HDDS-7322. Remove unused dependencies hsqldb and jettison
(#3834)
add 46e58a6350 HDDS-7300. Race condition between full data scan and block
deletion (#3811)
add 02341d77f3 HDDS-7269. Remove fair lock configuration for container
state management flows. (#3815)
add 35eedb6b36 HDDS-7303. EC: ECBlockReconstructedStripeInputStream should
set initialized only at the end of init() (#3816)
add c45b31a4f4 HDDS-7330. Cleanup of BucketManagerImpl (#3838)
add c0de6bcdeb HDDS-7261. Add container location cache metrics. (#3829)
add 554eb9eb54 HDDS-7337. Replace set-output in Github Actions workflow
(#3839)
add 9d05818a87 HDDS-7343. Do not log exception on file not found in
getFileStatus() (#3849)
add 2dc800dc80 HDDS-7214. Continuous start & stop can have hanging threads
in stopping (#3782)
add f7ba4ba588 HDDS-7340. Bump jackson-databind to 2.13.4.2 (#3845)
add 02c266d3fe HDDS-6893. EC: ReplicationManager - move the empty
container handling into RM from Legacy (#3831)
add 11f4686ce8 HDDS-7254. Document that moving SCM from non-HA to HA is
currently unsupported. (#3846)
add 03e670b865 HDDS-7149. Update ratis version to 2.4.0 and thirdparty
version to 1.0.2. (#3855)
add a6316c8320 HDDS-7352. OM log flooded by AWSV4AuthValidator (#3857)
add 237a9a1594 HDDS-7058. EC: ReplicationManager - Implement ratis
container replication check handler (#3802)
add 977ab59e11 HDDS-7341. EC: Close pipelines with unregistered nodes
(#3850)
add d5dc65eaa0 HDDS-7305. Fix Hadoop imports (#3822)
add d32e96c835 HDDS-7351. Use jackson-bom to ensure consistent Jackson
version (#3856)
add e45f9b8333 HDDS-7199. Implement new mix workload Read/Write Freon
command which meets specific test requirements (#3754)
add ff6d15f5df HDDS-7354. SchemaV3 blockData not deleted in table (#3860)
add 3fd7cd0896 Revert "HDDS-7199. Implement new mix workload Read/Write
Freon command which meets specific test requirements (#3754)"
add 1fa6d02e8a HDDS-6930. SCM,OM,RECON should not print ERROR and exit
with code 1 on successful shutdown (#3848)
add df48ca4fe3 HDDS-7356. Update SCM-HA.zh.md to match the English version
(#3861)
add 31560fcdfe HDDS-7355. non-primordial scm fail to get signed cert from
primordial SCM when converting an unsecure cluster to secure (#3859)
add f9b74a25cf HDDS-6210. EC: Add EC metrics (#3851)
add 5c2a39365a HDDS-7369. Fix wrong order of command arguments in
Nonrolling-Upgrade.md (#3866)
add 13a6d0178e HDDS-7141. Recon: Improve Disk Usage Page (#3789)
add ae59f8ae5f HDDS-7248. Recon: Expand the container status page to show
all unhealthy container states (#3837)
add ecdfc20756 HDDS-7199. Implement new mix workload Read/Write Freon
command (#3872)
add fdc57a93cb HDDS-7403. README Security Improvement (#3879)
add df0d1e81e3 HDDS-7368. [Multi-Tenant] Add Volume Existence check in
preExecute for OMTenantCreateRequest (#3869)
add 2f1117539c HDDS-7284. JVM crash for rocksdb for read/write after close
(#3801)
add 74aef20c9e HDDS-7182. Add property to control RocksDB max open files
(#3843)
add b9a47f6bbf HDDS-7253. Fix exception when '/' in key name (#3774)
add 340f3a7876 HDDS-7381. Cleanup of VolumeManagerImpl (#3873)
add dfc13a0506 HDDS-7258. Cleanup the allocated but uncommitted blocks
(#3778)
add 5b7f448a53 HDDS-7121. Support namespace summaries (du, dist & counts)
for legacy FS buckets (#3746)
add 462f32dd69 HDDS-7396. Force close non-RATIS containers in
ReplicationManager (#3877)
add 561788ebe7 Revert "HDDS-7253. Fix exception when '/' in key name
(#3774)"
add 9449747ff7 HDDS-7413. Fix logging while marking container state
unhealthy (#3887)
add a664ccac6a HDDS-7342. Move encryption-related code from
MultipartCryptoKeyInputStream to OzoneCryptoInputStream (#3852)
add b5ecea61de HDDS-7349. Flaky integration test have memory leak for
RatisDropwizardExports (#3858)
add 0652ba433b HDDS-7422. Bump woodstox-core from 5.0.3 to 5.4.0 (#3886)
add 557d7f84ff HDDS-7402. Adapt CommandQueue to track the count of each
queued command type (#3891)
add 3fb0dfb464 HDDS-7407. EC: Block allocation should not be stripped
across the EC group (#3882)
add 901bcf2dcc HDDS-7424. Bump jetty to 9.4.49.v20220914 (#3894)
add a84baced7e HDDS-7220. SCM should use sub-ca certificate for token
signature without HA enabled (#3752)
add 880f87cd4e HDDS-7370. Add pending commands in SCM to Datanode command
count (#3867)
add f6cad7c55f HDDS-7328. Improve Deletion of FSO Paths (#3844)
add dd67faa0b0 HDDS-7406. Remove unused the exception & improve debug log
in KeyDeletingService (#3880)
add 3a52215bb7 HDDS-1157. TestOzoneContainerWithTLS is failing with
SSLHandshakeException (#3888)
add f0d41c5fcb HDDS-7316. Print stacktrace to identify the location of
RocksObject leaks. (#3827)
add bd8a1610b3 HDDS-7421. Respect OZONE_LOGLEVEL and OZONE_ROOT_LOGGER for
CLI commands (#3892)
add 30d1a4e359 HDDS-7420. Bump Spring framework from 5.2.20 to 5.3.23
(#3902)
add cc574c6c5f HDDS-7432. Move command summary into Commands object in
CommandQueue (#3901)
add 7394f2c2ae HDDS-7361. Add general metrics for queues in Datanode
(#3863)
add c187a8ff65 HDDS-7384. EC: ReplicationManager - implement deleting
container handler (#3881)
add 3294d28cc9 HDDS-7231. Integrate the GetKeyInfo API to key read flows
(#3800)
add 416c575685 HDDS-7146. Recon: Add button to trigger OM DB Sync
(Backend) (#3821)
add f98376b743 HDDS-7244. Fix multiple reports queued up from same DN and
using up heap (#3783)
add 9c6d6a9da5 HDDS-7290. provide a config to increase the list batch size
in OzoneFileSystem (#3853)
add 2e1de7b95f HDDS-7363. Changes to Ozone CLI to validate if the jars in
classpath files are present on an install (#3864)
add 03595deac6 HDDS-7429. Remove Tag from SCMMetadataStoreMetrics (#3897)
add c71e1da783 HDDS-7320. Enable Container SchemaV3 by default. (#3841)
add c7f1c7c9ee HDDS-7447. Leak of TableIterator in
DirectoryDeletingService. (#3922)
add 8dded6ce7e HDDS-6926. Add support for shaded protobufs used by
hadoop-client/spark. (#3915)
add a77cc1da90 HDDS-7435. ContainerOperationClient cleanup (#3910)
add 074d225f2a HDDS-7025. Add cache metrics in OM. (#3865)
add 891e9f57a9 HDDS-7090. EC: delete empty closed EC container (#3660)
add d004f69b98 HDDS-6757. Test "size.leaving.source.max" limit in
ContainerBalancer (#3904)
add b316e73095 HDDS-7423. DB Scanner should allow iteration from specific
key (#3893)
add 5dd389b7cb HDDS-5866. Discrepancy in Trash directory in ofs vs o3fs.
(#3906)
add 6fed1c9434 HDDS-7324. S3G: Avoid calling getS3VolumeContext by using
GetGeyInfo API. (#3911)
add 2037968a56 HDDS-7419. Integrate the GetKeyInfo API to OFS (#3912)
add 3c0ec81687 HDDS-7362. Bring back the mechanism to share LRU cache
between RocksDB in datanodes (#3921)
add 5453cd6aec HDDS-3419. Throw exception with correct code when available
data nodes are not sufficient (#3847)
add 4ec46bc102 HDDS-7460. Bump snakeyaml from 1.32 to 1.33 (#3935)
add 715a3f6d06 Upgrade RocksDB to 7.7.3 (#3924)
add f95da6c66e HDDS-7451. Add UT of SyncDataFromOM for
SequenceNumberNotFoundException (#3927)
add 14a1668521 HDDS-7326. Intermittent timeout in
TestECContainerRecovery.testContainerRecoveryOverReplicationProcessing (#3941)
add a499123287 HDDS-6896. EC: ReplicationManager - Add relevant metrics to
the various ReplicationManager classes (#3909)
add f9df0593e4 HDDS-5112. Recon Insights page does not list buckets when
only one volume is present. (#3907)
add 436b4b21eb HDDS-2642. Expose decommission / maintenance metrics via
JMX (#3781)
add 39dfe7ad86 HDDS-7453. Check certificate expiration at service startup,
renew if necessary. (#3930)
add 0af369e25f HDDS-7440. Delete package-info class for
org.apache.hadoop.io.erasurecode.rawcoder to avoid conflict with Hadoop (#3925)
add 3d3e6cb19a HDDS-7314. Recon graceful shutdown while closing RocksDB
handle. (#3931)
add 9a25a8ea7e HDDS-7360. Investigate TestRootedOzoneFileSystem test
cases. (#3928)
add 39747954a6 HDDS-7323. Recon: Auto refresh toggle is switched back when
visiting new site (#3868)
add fa5695b677 HDDS-7456 Recon: Refreshed at not working on Pipeline and
DataNode page (#3933)
add 03fa68109a HDDS-7408. Recon: Refactor container status page (#3916)
add ec0d24a9da HDDS-4385. Search function using container ID on Recon
Missing Container (#3917)
add 4dc3083954 HDDS-7469. Recon: Add more information to Open Containers
column on on the Datanodes page. (#3944)
add c4e9e348fa HDDS-7446. Cleanup possibly onDisk unreferenced blocks
(#3923)
add a13c62b605 HDDS-7471. EC: Notify ReplicationManager when a heartbeat
updates datanode command counts (#3945)
add a8a0a4533c HDDS-7475. Remove outdated Hadoop dependencies (#3950)
add b3e7f3c320 HDDS-7485. Bump commons-codec to 1.15 (#3955)
add 73d9082168 HDDS-7468. Recon: clusterState API Changes for adding
missingContainerCount (#3952)
add 0f2f15b79f HDDS-7490. fix metaTable error (#3961)
add b46f961b59 HDDS-7255. Add metrics for container reports events (#3936)
add 006df10d67 HDDS-7101. EC: ReplicationManager - handle UNHEALTHY
replicas (#3962)
add 8518ed1cff HDDS-6971. EC: Add EC block checksum computer. (#3790)
add 95568ab735 HDDS-7095. allow on demand scanning for containers (#3788)
add ba41ef5041 HDDS-7487. Support fine grained certificate lifetime for
efficient test. (#3954)
add 7e38bfe164 HDDS-7488. EC: ReplicationManager: Move Mis-Replicated into
a separate unhealthy state (#3956)
add 559ad868ef HDDS-4402. Recon dashboard page does not load until missing
containers API returns data. (#3968)
add 412a13aa56 HDDS-7245. Recon: Add button to trigger OM DB Sync
(Frontend) (#3929)
add 1bc9c8a0ac HDDS-7474. Recon: Add count to containers page tab label
(#3951)
add d476389a4b HDDS-7260. dfs -du should return correct disk space
consumed by all replicas (#3810)
add 6ff33075f8 HDDS-7470. [Quota] bucket is created crossing quota makes
OM crash and unable to start (#3960)
add 6a0d4d5c7f HDDS-7346. Cannot set bucket args when the bucket has quota
set (#3975)
add 85e7cd1867 HDDS-7498. Add permission check when `--user` is specified
in `ozone sh volume list` (#3971)
add 5e23afee10 HDDS-4565. [FSO]Delete : quotaReleased on directory
deletion should consider all sub paths (#3973)
add edefe79434 HDDS-7476. SCM root CA stored under hdds.datanode.dir if
ozone.metadata.dirs is not specified (#3948)
add 0ed0240c66 HDDS-7280. New test addition to
TestOMBucketCreateRequestWithFSO (#3794)
add 0f0645ed23 HDDS-5526. ContainerBalancer#checkConditionsForBalancing
pre-emptively checks iteration limits. (#3895)
add 99493d52f8 HDDS-7518. Fix intermittent failure in ozonesecure
replication test (#3986)
add 1e86860e1b HDDS-7462. EC: Fix Reconstruction Issue with
StaleRecoveringContainerScrubbingService (#3939)
add cb9b5d06dd HDDS-7092. EC: Offline Recovery with simultaneous Over
Replication & Under Replication (#3984)
add c8648b159e HDDS-7528. EC: ReplicationManager - refactor logic to send
datanode commands into a central place (#3990)
add 1370de18b8 HDDS-7532. EC: ReplicationManager - remove calls to
ECHealthCheck from under and over replication processing (#3995)
add 864397d1c6 HDDS-7187. EC: Retry failed writes before rewrite to a new
block group (#3977)
add b92f7c58f5 HDDS-7481. Set quota with reduced size than actual used do
not fail (#3978)
add 302097fabb HDDS-7353. Cleanup of unused code in SCM (#3905)
add 6abee8a48c HDDS-7534. Intermittent failure in
TestKeyDeletingService#checkDeletionForEmptyKey (#3998)
add 7f3f28c54e HDDS-7538. Remove unused OmUtils#getMD5Digest (#3996)
add 4145cf5a08 HDDS-7449. Avoid overwriting bucket encryption properties
when quota/replication config are set on an existing bucket (#3992)
add 6570c19081 HDDS-7535. Eliminate duplicated config in
LegacyReplicationManager (#4000)
add 7af8d06d87 HDDS-7289. Bump protobuf-java from 3.19.2 to 3.19.6 (#4003)
add 27ff3b6442 HDDS-7486. Support KeyStoreFactory which supports
keyManager and trustManager reload (#3959)
add e50c443d24 HDDS-7536. Remove unused dependency declarations (#4001)
add 8db02c864b HDDS-7500. Ozone recon server Memory Optimization. (#3974)
add f552cf21a1 HDDS-7540. Separate scheduled CI from push/PR workflows
(#4004)
add 829dfce410 HDDS-7554. Recon UI should show DORMANT in pipeline status
filter (#4010)
add eeeb0b548c HDDS-6615. EC: Improve write performance by pipelining
encode and flush (#3994)
add f1d651d0b5 HDDS-7561. Improve setquota, clrquota CLI usage (#4016)
add 627c90c233 HDDS-7510. Recon: Return number of open containers in
`/clusterState` endpoint (#3989)
add 4de7d56fc4 HDDS-7394. OM RPC FairCallQueue decay decision metrics list
caller username in the metric (#3878)
add 9a5bfb4fbd HDDS-7568. Bump copyright year in NOTICE (#4018)
add 45778cc699 HDDS-7540. Let reusable workflow inherit secrets (#4012)
add cb9caca398 HDDS-7575. Correct showing of RATIS-THREE icon in Recon UI
(#4026)
add fe86f3bb49 HDDS-7567. Invalid entries in LICENSE (#4020)
add 50a9d46d80 HDDS-7497. Fix mkdir does not update bucket's usedNamespace
(#3969)
add be092c4153 HDDS-7563. Add a handler for under replicated Ratis
containers in RM (#4025)
add 984d26a165 HDDS-7562. Suppress warning about long filenames in tar
(#4017)
add 1119f9d949 HDDS-7579. Use Netty 4.1.77 for consistency (#4031)
add 069e038c4f HDDS-7253. Fix exception when '/' in key name (#4038)
add 47a5671cc5 HDDS-7581. Fix update-jar-report for snapshot (#4034)
add b532d2f7f5 HDDS-7541. FSO recursive delete directory with hierarchy
takes much time for cleanup (#4008)
add ee2bd17440 HDDS-7555. Upgrade Ratis to 2.4.2-8b8bdda-SNAPSHOT. (#4028)
add a1bc9b86de HDDS-5704. Ozone URI syntax description in help content
needs to mention about ozone service id (#3862)
add 037492f839 HDDS-7472. EC: Fix NSSummaryEndpoint#getDiskUsage for EC
keys (#3987)
add 721b02593b HDDS-7526. Avoid overwriting replication config on existing
bucket when quota is set (#4013)
add fd3ddc42b2 HDDS-7599. Bump commons-net from 3.6 to 3.9.0 (#4046)
add d50a9dd894 HDDS-7550. Recon: UI sorting not functioning as expected
(#4009)
add 565775c7dd HDDS-7464. Container Report at SCM is not coming separately
for ICR and FCR in prometheus endpoint (#4035)
add a93ca68066 HDDS-7597. Clarify SCM admin access denied message (#4056)
add 8967738e93 HDDS-7492. Extend Placement Policy Interface to select
mis-replicated replicas to copy (#4006)
add ebd1c747ac HDDS-7615. Import Guava classes directly, not from
hadoop-thirdparty (#4062)
add 77d386de5f HDDS-7502. Recon: Show number of open containers on the
Overview page (#4043)
add 6eed1ec3d3 HDDS-7265. ScatterRackPolicy for Ratis pipeline provider
container placement (#4002)
add 845ceff4d6 HDDS-7603. Enable Erasure coding and encryption flag on
FileStatus for ofs (#4060)
add a9bb08889c HDDS-7587. Update NOTICE and LICENSE in license/bin/ (#4039)
add e7567f8958 HDDS-7617. Mark testECContainerRecoveryWithTimedOutRecovery
as flaky
add 66e277386e HDDS-7572. Use keyManager and trustManager provided by
keyStoreFactory in datanode grpc services (#4032)
add 76fb0163df HDDS-7580. Add option to show key count in DBScanner.
(#4033)
add d0e6824eec HDDS-7614. Add subscription mechanism to
ContainerReplicaPendingOps (#4064)
add c9a0e2cd10 HDDS-7589. Unify multipart stream reading. (#4041)
add 3fe4b4e156 HDDS-7595. Add OmId in OM's UI (#4047)
add e4a1993e79 HDDS-7618. Replication Commands should timeout if not
processed on datanodes in time (#4069)
add a6307d71d0 HDDS-7628. Disable flaky
TestOzoneContainerWithTLS#testContainerDownload
add 4c529ebc11 HDDS-7591. Add a handler for over replicated Ratis
containers to RM (#4055)
add d6f63bf2e5 HDDS-7619. Update SCM term in datanode when command is
received (#4072)
add 2279689dd3 HDDS-7008. Mark TestOzoneHARatisLogParser as flaky (#4078)
add 551ef3c020 HDDS-7640. EC: UNHEALTHY replicas not replaced by healthy
replicas from a CLOSED container by RM (#4083)
add 2fc94fa05b HDDS-7582. ECUnderReplicationHandler does not consider
pending adds when finding targets (#4085)
add 6064b84576 HDDS-7503. CLI admin namespace -quota should return whole
file system disk capacity consumed by all replicas (#3976)
add 2ba8bb71f1 HDDS-7643. rocksdb iterator is not closed properly (#4084)
add a756937a16 HDDS-7627. EC: Bug fix for calculating Misreplication Count
(#4088)
add 5eead92666 HDDS-7592. Recursive delete of directory multiple time
impacts some directory not removed (#4068)
add fd0eddda8c HDDS-7642. EC: Add debug logging to the Replication Manager
check handlers (#4087)
add b347fde11b HDDS-7628. Disable flaky
TestOzoneContainerWithTLS#testCreateOzoneContainer
add 2d4bc6a3e2 HDDS-7636. Remove hadoop security dependency in
org.apache.hadoop.hdds.security.ssl package (#4080)
add 1a5e698c6b HDDS-7521. EC: Misreplication Handler changes for Placement
Policy interface changes (#4061)
add 1f7e9cc8ed HDDS-7620. Check term for async commands before processing
(#4096)
add 517e3e15c5 HDDS-7371. Create properties for all dependency versions
(#4059)
add fdf219f179 HDDS-5366. [Ozone-Streaming] Implement stream method to
ContainerStateMachine. (#2358). Contributed by mingchao zhao
add 0afbbc541d HDDS-5452. Add link method to ContainerStateMachine for
Ratis streaming (#2422)
add 9eafca6ee8 HDDS-5481. Fix stream() and link() method in
ContainerStateMachine. (#2451)
add da95602eba HDDS-5480. [Ozone-Streaming] Client and server should
support stream setup. (#2452)
add 82153a26ed HDDS-5488. [Ozone-Streaming] Add a new
BlockOutputStream/KeyOutputStream to support streaming api (#2495)
add 5a294b7d54 HDDS-5599. [Ozone-Streaming]drop BufferPool and
ChunkBuffer to avoid buffer copying (#2557)
add ff59eeb27f HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to
ByteBufferStreamOutput (#2603)
add f27388a5cc HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
add 94f87891a7 HDDS-5486. [Ozone-Streaming] Streaming supports writing in
Pipline mode (#2682)
add 9d01c02c98 HDDS-5849. [Ozone-Streaming]Write exceptions occur after
checksum is enabled (#2729)
add 122f96c029 HDDS-5674.[Ozone-Streaming] Handle client retries on
exception (#2701)
add e063706ed3 HDDS-5895. [Ozone-Streaming] Make
raft.server.data-stream.client.pool.size configurable (#2766)
add 75efaae8cf HDDS-5763. Provide an Executor for each LocalStream in
ContainerStateMachine (#2782)
add 2e86e93321 HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream
config (#2841)
add 9a7edb0240 HDDS-5961. [Ozone-Streaming] update the usage space of
Containers in the stream write (#2833)
add 237e0afed3 HDDS-5879. [Ozone-Streaming] OzoneBucket add the
createMultipartStreamKey method (#2760)
add 95e317b551 HDDS-5743. [Ozone-Streaming] Add option to write files via
streaming api in ofs and o3fs. (#2770)
add 9a2de9a530 HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer
fixed boundary for streaming writes. (#2866)
add ac9921d65d HDDS-6039. Define a minimum packet size during streaming
writes. (#2883)
add a67569fe83 HDDS-6130. [Ozone-Streaming] When releaseBuffers will get
“Couldn 't find the required future” (#2939)
add f85b42dcee HDDS-6139. [Ozone-Streaming] Fix incorrect computation of
totalAckDataLength. (#2978)
add 0e8c8004a1 HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
add 2657f5ad95 HDDS-6138.[Ozone-Streaming] Define a limit on the size of
the retry bufferList. (#2946)
add 14ffbd9cb1 HDDS-6298. Add XceiverServerRatis stream config (#3070)
add 872e3061fe HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support
FlushDelay. (#3002)
add 92054a4d2f HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE
(#3060)
add 8a5f2d6192 HDDS-6229. [Ozone-Streaming] Data Channel abstraction on
datanode (#3023)
add 8f68101eb8 HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
add bb3af31757 HDDS-6388. [Ozone-Streaming] Streaming write support both
pipeline model and star model (#3145)
add dfefdf20eb HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters.
(#3207)
add 95dbea64a0 HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput.
(#3195)
add 6a6384233e HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at
the end of the stream. (#3229)
add 3d821e31ff HDDS-5666. Add option to createKey via streaming api in
Freon (#2574)
add 7f22916889 HDDS-6592. [Ozone-Streaming] Fix
ContainerStateMachine#applyTransaction assert error (#3315)
add 45954da28b HDDS-6842. [Ozone-Streaming] Reduce the number of watch
requests in StreamCommitWatcher. (#3492)
add 7184025ffb HDDS-6867. [Ozone-Streaming] PutKeyHandler should not use
streaming to put EC key. (#3516)
add 07bf1dbcf0 HDDS-6955. [Ozone-streaming] Add explicit stream flag in
ozone shell (#3559)
add 13da2a7c4a HDDS-7431. [Ozone-Streaming] Disable data steam by default.
(#3900)
add 8a0e347105 HDDS-7438. [Ozone-Streaming] Add a createStreamKey method
to OzoneBucket. (#3914)
add 9bbf5a1ada HDDS-7425. Add documentation for the new Streaming Pipeline
feature. (#3913)
add 1e155f3820 HDDS-7478. [Ozone-Streaming] NPE in when creating a file
with o3fs. (#3949)
add 0650ad9b08 HDDS-7426. Add a new acceptance test for Streaming
Pipeline. (#4019)
add 8b2ca00822 HDDS-7616. EC: Refactor Unhealthy Replicated Processor
(#4063)
add 886733b373 HDDS-7605. Improve logging in Container Balancer (#4067)
add ca2d59bde4 HDDS-6650. S3MultipartUpload support update bucket
usedNamespace. (#4081)
add 2ac31e1bcb HDDS-7398. Tool to remove old certs from the scm db (#3972)
add 1235e9b3d1 HDDS-7399. Enable specifying external root ca (#4053)
add d93dcd396f HDDS-7649. S3 multipart upload EC release space quota wrong
for old version (#4095)
add d5f917274c HDDS-7621. Update SCM term in datanode from heartbeat
without any commands (#4101)
add ef480765d8 HDDS-7654. EC: ReplicationManager - merge mis-rep queue
into under replicated queue (#4099)
add 7ca9b5dfa6 HDDS-7565. FSO purge directory for old bucket can update
quota for new bucket (#4021)
add bdeceeedad HDDS-7635. Update failure metrics when allocate block fails
in preExecute. (#4086)
add 204f8e18c5 HDDS-7683. EC: ReplicationManager - UnderRep maintenance
handler should not request nodes if none needed (#4109)
add c7785fa4fd HDDS-7463. SCM Pipeline scrubber never able to cleanup
allocated pipeline. (#4093)
add 9077cb83d1 HDDS-7473. Ratis integration for support of remove
registeration (#4112)
add add8f8bb16 HDDS-7686. Cherry-pick proto.lock files change from
ozone-1.3 release branch to master (#4113)
add 157c591b13 HDDS-7631. Log format error on quotas when exceeding the
space quota (#4076)
add fa8494cd1d HDDS-5331. Recon: Trigger PipelineSyncTask when DN becomes
stale and ContainerHealthTask when DN becomes dead. (#4011)
add 58283b6230 HDDS-7684. Embed Matomo Web Analytics tracking code in
docs. (#4111)
add c40cb07053 HDDS-7576. Prometheus metrics do not remove stale metrics
until restart (#4057)
add a7f06c1964 HDDS-3486. Recon cannot track missing containers that were
created and went missing while it is down. (#3947)
add b167d99d73 HDDS-7692. Remove usage of MonotonicClock and use default
system clock instead (#4121)
add b828881f78 HDDS-7661. Ratis Misreplication Handler (#4100)
add bfc37bb886 HDDS-7641. Namespace quota validation is not present in
multiple places (#4082)
add 98eb2ebd99 HDDS-7644. S3 multipart upload does not update quota
namespace for missing parents (#4091)
add a89ba1a4c4 HDDS-7584. Addition of new OM node expels itself from the
Ratis ring after restart (#4074)
add 7cfaf96b81 HDDS-7525. Migrate key digest from MD5 to SHA256 in Ozone
shell
add 4184499e80 HDDS-7623. Do not compress container re-replication traffic
by default (#4089)
add b352ad03ab HDDS-7409 [doc] Update documents for better presentation
(#3884)
add 1c830c5bc7 HDDS-7564. Update Ozone version after 1.3.0 release (#4115)
add eb031df2ed HDDS-7648. Add a servername tag in UGI metrics. (#4094)
add 04f68c96f4 HDDS-7633. Compile error with Java 11: package
com.sun.jmx.mbeanserver is not visible (#4077)
add c119557241 HDDS-7588. Intermittent failure in
TestObjectStoreWithLegacyFS#testFlatKeyStructureWithOBS (#4040)
add 58da8a90a1 HDDS-7716. Log read requests rejected with permission
denied in OM audit (#4136)
add 21717321c6 HDDS-7714. Docker cluster ozone-om-ha fails during
docker-compose up (#4137)
add 9b48d8b1c0 HDDS-7696. MisReplicationHandler does not consider
QUASI_CLOSED replicas as sources (#4144)
add 41f048804a HDDS-7339. Implement Certificate renewal task for services
(#3982)
add 10811c597f HDDS-7666. EC: Unrecoverable EC containers with some
remaining replicas may block decommissioning (#4118)
add 9d30c9a7ce HDDS-7606. Add SCM HA support in intellij run (#4058)
add 7d6ee034d7 HDDS-7727. EC: SCM unregistered event handler for
DatanodeCommandCountUpdated (#4147)
add ecc2f189e4 HDDS-7708. No check for certificate duration config
scenarios. (#4149)
add f2f0afa9f6 HDDS-7243. Remove RequestFeatureValidator from echoRPC
method which supports only ValidationCondition.OLDER_CLIENT_REQUESTS (#4051)
add 2eb5805b6c HDDS-7738. SCM terminates when adding container to a closed
pipeline (#4154)
add e09f9a208a HDDS-7729. EC: ECContainerReplicaCount should handle
pending delete of unhealthy replicas (#4146)
add 4abe983acf HDDS-7695. EC metrics related to replication commands don't
add up (#4152)
add a1cb9d567d HDDS-7628. Intermittent failure in
TestOzoneContainerWithTLS (#4142)
add dcfd621e15 HDDS-7652. Volume Quota not enforced during write when
bucket quota is not set (#4124)
add 07b67597e8 HDDS-7739. EC: Increase the information in the RM sending
command log message (#4153)
add a80c6b1c73 HDDS-7726. EC: Enhance datanode reconstruction log message
(#4155)
add b6cda5162f HDDS-4539. Container Health Task should not run until Recon
has reached steady state. (#4049)
add ed7c60cafd HDDS-7560. Placement Policy Interface changes to handle
Overreplication (#4014)
add ab91e46247 HDDS-7638. Ozone client change to support HSync. (#4104)
new d8765436c2 Merge remote-tracking branch 'asf/master' into
HDDS-6517-Snapshot
add cbbddc6fdc HDDS-7630. Show detailed OMResponse when
OzoneManagerDoubleBuffer terminates (#4075)
add dff829623b HDDS-7705. Fix OM Bootstrap request (#4160)
add 4531701c4d HDDS-7753. Simplify DatanodeDetails#toString to improve log
messages (#4161)
new 98e3a5033d Merge remote-tracking branch 'asf/master' into
HDDS-6517-Snapshot
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/{post-commit.yml => ci.yml} | 86 +-
.github/workflows/close-pending.yaml | 2 +-
.github/workflows/comments.yaml | 2 +-
.github/workflows/post-commit.yml | 405 +--
.../{close-pending.yaml => scheduled_ci.yml} | 20 +-
LICENSE.txt | 10 +-
NOTICE.txt | 2 +-
README.md | 2 +-
dev-support/ci/lib/_initialization.sh | 4 +-
hadoop-hdds/annotations/pom.xml | 4 +-
hadoop-hdds/client/pom.xml | 4 +-
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 56 +-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 74 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 21 +-
.../hadoop/hdds/scm/XceiverClientMetrics.java | 10 +
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 +-
.../hdds/scm/storage/AbstractDataStreamOutput.java | 131 +
...utputStream.java => BlockDataStreamOutput.java} | 691 +++--
.../hdds/scm/storage/BlockExtendedInputStream.java | 11 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 50 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 12 +-
.../hdds/scm/storage/ByteBufferStreamOutput.java | 57 +
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 19 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 98 +
.../hdds/scm/storage/MultipartInputStream.java | 246 ++
...tendedInputStream.java => PartInputStream.java} | 22 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 16 +-
.../hadoop/hdds/scm/storage/StreamBuffer.java} | 48 +-
.../hdds/scm/storage/StreamCommitWatcher.java | 208 ++
.../hadoop/ozone/client/io/ECBlockInputStream.java | 5 -
.../ozone/client/io/ECBlockInputStreamProxy.java | 9 +-
.../client/io/ECBlockReconstructedInputStream.java | 2 +-
.../io/ECBlockReconstructedStripeInputStream.java | 13 +-
.../hdds/scm/storage/TestBlockInputStream.java | 133 +-
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 5 -
.../ozone/client/io/TestECBlockInputStream.java | 5 -
.../TestECBlockReconstructedStripeInputStream.java | 20 +
hadoop-hdds/common/pom.xml | 6 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 45 +
.../hadoop/hdds/client/ReplicationConfig.java | 11 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 13 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 92 +-
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 3 +
.../apache/hadoop/hdds/scm/PlacementPolicy.java | 28 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 20 +-
.../scm/container/common/helpers/ExcludeList.java | 4 +-
.../protocol/StorageContainerLocationProtocol.java | 7 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 14 +-
.../hadoop/hdds/security/x509/SecurityConfig.java | 140 +-
.../x509/certificate/utils/CertificateCodec.java | 5 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 52 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 3 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../org/apache/hadoop/ozone/audit/DNAction.java | 3 +-
.../org/apache/hadoop/ozone/common/Checksum.java | 5 +
.../apache/hadoop/ozone/common/StorageInfo.java | 4 +
.../ozone/container/common/helpers/ChunkInfo.java | 14 +
.../helpers/ContainerCommandRequestPBHelper.java | 1 +
.../java/org/apache/hadoop/util/CacheMetrics.java | 103 +
.../org/apache/hadoop/util/CheckedRunnable.java | 25 +-
.../org/apache/hadoop/util/CheckedSupplier.java | 29 +
.../java/org/apache/hadoop/util/MetricUtil.java | 52 +
.../common/src/main/resources/ozone-default.xml | 192 +-
.../hadoop/ozone/audit/AuditLogTestUtils.java | 79 +
.../ozone/container/ContainerTestHelper.java | 30 +-
hadoop-hdds/config/pom.xml | 4 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +-
hadoop-hdds/container-service/pom.xml | 17 +-
.../hdds/datanode/metadata/CRLDBDefinition.java | 5 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 141 +-
.../container/common/helpers/ContainerUtils.java | 29 +
.../common/impl/ContainerLayoutVersion.java | 26 +-
.../ozone/container/common/impl/ContainerSet.java | 3 +-
.../container/common/impl/HddsDispatcher.java | 66 +-
.../common/interfaces/ContainerDispatcher.java | 10 +
.../ozone/container/common/interfaces/Handler.java | 16 +
.../container/common/report/ReportPublisher.java | 6 +-
.../common/statemachine/DatanodeConfiguration.java | 38 +-
.../common/statemachine/DatanodeQueueMetrics.java | 179 ++
.../common/statemachine/DatanodeStateMachine.java | 30 +-
.../common/statemachine/EndpointStateMachine.java | 1 +
.../common/statemachine/StateContext.java | 59 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 19 +-
.../DeleteContainerCommandHandler.java | 67 +-
.../ReconstructECContainersCommandHandler.java | 6 +-
.../ReplicateContainerCommandHandler.java | 3 +-
.../states/endpoint/HeartbeatEndpointTask.java | 7 +
.../common/transport/server/XceiverServerGrpc.java | 4 +-
.../server/ratis/ContainerStateMachine.java | 79 +-
.../common/transport/server/ratis/LocalStream.java | 58 +
.../transport/server/ratis/XceiverServerRatis.java | 43 +-
.../container/common/utils/DatanodeStoreCache.java | 29 +-
.../common/utils/db/DatanodeDBProfile.java | 16 +-
.../reconstruction/ECContainerOperationClient.java | 40 +-
.../ECReconstructionCommandInfo.java | 81 +-
.../ECReconstructionCoordinator.java | 77 +-
.../ECReconstructionCoordinatorTask.java | 66 +-
.../ec/reconstruction/ECReconstructionMetrics.java | 80 +
.../reconstruction/ECReconstructionSupervisor.java | 11 +-
.../container/keyvalue/KeyValueContainer.java | 9 +-
.../container/keyvalue/KeyValueContainerCheck.java | 108 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 104 +
.../container/keyvalue/TarContainerPacker.java | 34 +-
.../container/keyvalue/helpers/BlockUtils.java | 2 +-
.../keyvalue/impl/ChunkManagerDispatcher.java | 16 +
.../keyvalue/impl/FilePerBlockStrategy.java | 20 +
.../keyvalue/impl/KeyValueStreamDataChannel.java | 277 ++
.../keyvalue/impl/StreamDataChannelBase.java | 96 +
.../keyvalue/interfaces/ChunkManager.java | 13 +
.../background/BlockDeletingService.java | 82 +-
.../StaleRecoveringContainerScrubbingService.java | 5 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 4 +
.../ozoneimpl/ContainerScannerConfiguration.java | 29 +-
.../ozoneimpl/OnDemandContainerScanner.java | 178 ++
.../ozoneimpl/OnDemandScannerMetrics.java | 42 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 57 +-
.../replication/ContainerReplicationSource.java | 5 +-
.../replication/CopyContainerCompression.java | 78 +
.../replication/GrpcReplicationClient.java | 20 +-
.../replication/GrpcReplicationService.java | 8 +-
.../OnDemandContainerReplicationSource.java | 19 +-
.../container/replication/ReplicationServer.java | 26 +-
.../replication/ReplicationSupervisor.java | 62 +-
.../replication/ReplicationSupervisorMetrics.java | 5 +-
.../container/replication/ReplicationTask.java | 34 +-
.../replication/SimpleContainerDownloader.java | 12 +-
.../protocol/commands/CloseContainerCommand.java | 10 +
.../protocol/commands/DeleteContainerCommand.java | 16 +
.../commands/ReconstructECContainersCommand.java | 15 +
.../commands/ReplicateContainerCommand.java | 10 +
.../hadoop/ozone/protocol/commands/SCMCommand.java | 35 +
.../metadata/TestDatanodeCRLStoreImpl.java | 24 +-
.../hadoop/ozone/TestHddsDatanodeService.java | 16 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 315 ++-
.../ozone/container/common/ContainerTestUtils.java | 39 +-
.../container/common/TestBlockDeletingService.java | 200 +-
.../common/TestContainerLayoutVersion.java | 4 +-
.../common/TestDatanodeLayOutVersion.java | 10 +-
.../container/common/TestDatanodeStateMachine.java | 60 +-
.../container/common/TestDatanodeStoreCache.java | 2 +-
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
...stStaleRecoveringContainerScrubbingService.java | 38 +-
.../common/helpers/TestContainerUtils.java | 4 +-
.../container/common/impl/TestHddsDispatcher.java | 68 +
.../container/common/report/TestReportManager.java | 2 +-
.../common/report/TestReportPublisher.java | 39 +-
.../statemachine/TestDatanodeConfiguration.java | 4 +-
.../common/statemachine/TestStateContext.java | 103 +-
.../TestDeleteContainerCommandHandler.java | 135 +
.../states/datanode/TestRunningDatanodeState.java | 8 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 103 +-
.../volume/TestCapacityVolumeChoosingPolicy.java | 31 +-
.../container/common/volume/TestHddsVolume.java | 10 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 35 +-
.../TestVolumeIOStatsWithPrometheusSink.java | 2 +-
.../TestECReconstructionSupervisor.java | 103 +-
.../container/keyvalue/TestKeyValueContainer.java | 192 +-
.../keyvalue/TestKeyValueContainerCheck.java | 4 +-
.../TestKeyValueContainerMarkUnhealthy.java | 2 +
.../container/keyvalue/TestKeyValueHandler.java | 10 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 15 +-
.../container/keyvalue/TestTarContainerPacker.java | 37 +-
.../container/keyvalue/helpers/TestChunkUtils.java | 20 +-
.../keyvalue/impl/AbstractTestChunkManager.java | 25 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 10 +-
.../keyvalue/impl/TestChunkManagerDummyImpl.java | 4 +-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 6 +-
.../keyvalue/impl/TestFilePerChunkStrategy.java | 8 +-
.../impl/TestKeyValueStreamDataChannel.java | 313 +++
.../TestContainerScannerConfiguration.java | 22 +-
.../ozoneimpl/TestContainerScannerMetrics.java | 95 +-
.../ozoneimpl/TestOnDemandContainerScanner.java | 157 ++
.../ReplicationSupervisorScheduling.java | 18 +-
.../replication/TestGrpcOutputStream.java | 16 +-
.../replication/TestMeasuredReplicator.java | 36 +-
.../replication/TestReplicationConfig.java | 4 +-
.../replication/TestReplicationSupervisor.java | 111 +-
.../replication/TestSimpleContainerDownloader.java | 19 +-
.../stream/TestDirstreamClientHandler.java | 34 +-
.../container/stream/TestStreamingServer.java | 18 +-
.../upgrade/TestDatanodeUpgradeToSchemaV3.java | 4 +
.../upgrade/TestDatanodeUpgradeToScmHA.java | 2 +-
.../TestReconstructionECContainersCommands.java | 27 +-
.../dev-support/checkstyle/suppressions.xml | 1 +
hadoop-hdds/docs/content/concept/Recon.md | 1 +
.../docs/content/feature/Nonrolling-Upgrade.md | 6 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 4 +-
hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 7 +
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 73 +-
.../content/feature/Streaming-Write-Pipeline.md | 136 +
hadoop-hdds/docs/content/interface/Ofs.md | 32 +-
hadoop-hdds/docs/content/interface/S3.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 2 +-
hadoop-hdds/docs/content/security/SecurityAcls.md | 9 +-
.../docs/content/security/SecurityAcls.zh.md | 7 +-
.../docs/content/security/SecurityWithRanger.md | 2 +-
.../docs/content/security/SecurityWithRanger.zh.md | 2 +-
hadoop-hdds/docs/pom.xml | 4 +-
.../themes/ozonedoc/layouts/partials/header.html | 22 +
hadoop-hdds/erasurecode/pom.xml | 4 +-
.../framework/dev-support/findbugsExcludeFile.xml | 4 -
hadoop-hdds/framework/pom.xml | 17 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 35 +
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 4 +-
...inerLocationProtocolClientSideTranslatorPB.java | 20 +
.../hadoop/hdds/security/ssl/KeyStoresFactory.java | 69 +
.../hdds/security/ssl/MonitoringTimerTask.java | 77 +
.../security/ssl/PemFileBasedKeyStoresFactory.java | 199 ++
.../hdds/security/ssl/ReloadingX509KeyManager.java | 178 ++
.../security/ssl/ReloadingX509TrustManager.java | 152 ++
.../hadoop/hdds/security/ssl/package-info.java | 21 +-
.../certificate/authority/DefaultCAServer.java | 116 +-
.../x509/certificate/client/CertificateClient.java | 55 +-
.../client/CommonCertificateClient.java | 16 +-
.../certificate/client/DNCertificateClient.java | 97 +-
.../client/DefaultCertificateClient.java | 650 ++++-
.../certificate/client/OMCertificateClient.java | 57 -
.../certificate/client/ReconCertificateClient.java | 104 +-
.../certificate/client/SCMCertificateClient.java | 24 +-
.../certificates/utils/SelfSignedCertificate.java | 38 +-
.../x509/exceptions/CertificateException.java | 8 +-
.../hadoop/hdds/security/x509/keys/KeyCodec.java | 16 +-
.../hdds/security/x509/keys/SecurityUtil.java | 34 +
.../hadoop/hdds/server/events/EventExecutor.java | 21 +
.../FixedThreadPoolWithAffinityExecutor.java | 219 +-
.../hadoop/hdds/server/events/IEventInfo.java | 13 +-
.../hdds/server/events/SingleThreadExecutor.java | 6 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 2 +-
.../hdds/server/http/PrometheusMetricsSink.java | 109 +-
.../hdds/server/http/RatisDropwizardExports.java | 86 +-
.../hadoop/hdds/utils/DecayRpcSchedulerUtil.java | 120 +
.../org/apache/hadoop/hdds/utils/MetricsUtil.java | 100 +
.../hdds/utils/PrometheusMetricsSinkUtil.java | 116 +
.../hadoop/hdds/utils/TableCacheMetrics.java | 92 +
.../apache/hadoop/hdds/utils/UgiMetricsUtil.java | 68 +
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 6 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 221 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 12 +-
.../apache/hadoop/hdds/utils/db/TableConfig.java | 4 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 8 +-
.../hadoop/hdds/utils/db/cache/CacheStats.java | 30 +-
.../hdds/utils/db/cache/CacheStatsRecorder.java | 47 +-
.../hadoop/hdds/utils/db/cache/FullTableCache.java | 16 +-
.../hdds/utils/db/cache/PartialTableCache.java | 13 +-
.../hadoop/hdds/utils/db/cache/TableCache.java | 6 +
.../db/managed/ManagedColumnFamilyOptions.java | 13 +
.../hdds/utils/db/managed/ManagedObject.java | 22 +-
.../utils/db/managed/ManagedRocksObjectUtils.java | 23 +-
.../ssl/TestPemFileBasedKeyStoresFactory.java | 255 ++
.../security/ssl/TestReloadingX509KeyManager.java | 83 +
.../ssl/TestReloadingX509TrustManager.java | 83 +
.../token/TestOzoneBlockTokenSecretManager.java | 4 +-
.../hdds/security/x509/CertificateClientTest.java | 104 +-
.../certificate/authority/TestDefaultCAServer.java | 101 +-
.../client/TestDefaultCertificateClient.java | 316 ++-
...tInit.java => TestDnCertificateClientInit.java} | 69 +-
.../x509/certificate/utils/TestCRLCodec.java | 9 +-
.../certificate/utils/TestCertificateCodec.java | 27 +-
.../x509/certificates/TestRootCertificate.java | 24 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 24 +-
....java => TestPrometheusMetricsIntegration.java} | 148 +-
.../hdds/utils/TestDecayRpcSchedulerUtil.java | 108 +
.../hdds/utils/TestPrometheusMetricsSinkUtil.java | 229 ++
.../hadoop/hdds/utils/TestUgiMetricsUtil.java | 63 +
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 31 +
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 44 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 8 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 8 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
.../src/main/proto/ScmAdminProtocol.proto | 1 +
.../interface-admin/src/main/resources/proto.lock | 679 +++--
hadoop-hdds/interface-client/pom.xml | 4 +-
.../src/main/proto/DatanodeClientProtocol.proto | 15 +-
.../interface-client/src/main/resources/proto.lock | 991 ++++++--
hadoop-hdds/interface-server/pom.xml | 4 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 4 +
.../interface-server/src/main/resources/proto.lock | 840 ++++--
hadoop-hdds/pom.xml | 4 +-
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 6 +-
hadoop-hdds/server-scm/pom.xml | 14 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 205 +-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 34 +
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 14 +-
.../scm/block/DeletedBlockLogStateManager.java | 3 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 2 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 2 +-
.../scm/container/CloseContainerEventHandler.java | 5 +-
.../hdds/scm/container/ContainerManager.java | 32 +-
.../hdds/scm/container/ContainerManagerImpl.java | 48 +-
.../hdds/scm/container/ContainerReportHandler.java | 7 +-
.../scm/container/ContainerStateManagerImpl.java | 6 +-
.../IncrementalContainerReportHandler.java | 5 +-
.../balancer/AbstractFindTargetGreedy.java | 54 +-
.../scm/container/balancer/ContainerBalancer.java | 1170 +--------
.../ContainerBalancerSelectionCriteria.java | 22 +-
...nerBalancer.java => ContainerBalancerTask.java} | 712 ++----
.../scm/container/balancer/FindSourceGreedy.java | 55 +-
.../scm/container/balancer/FindSourceStrategy.java | 11 +
.../FindTargetGreedyByNetworkTopology.java | 21 +
.../balancer/FindTargetGreedyByUsageInfo.java | 23 +
.../scm/container/balancer/FindTargetStrategy.java | 9 +
.../ContainerPlacementStatusDefault.java | 36 +-
.../algorithms/SCMContainerPlacementRackAware.java | 8 +
.../SCMContainerPlacementRackScatter.java | 138 +-
.../algorithms/SCMContainerPlacementRandom.java | 3 +-
.../AbstractOverReplicationHandler.java | 4 +-
.../replication/ContainerCheckRequest.java | 6 +-
.../replication/ContainerHealthResult.java | 72 +-
.../replication/ContainerReplicaCount.java | 3 +
.../replication/ContainerReplicaPendingOps.java | 105 +-
.../ContainerReplicaPendingOpsSubscriber.java | 33 +-
.../DatanodeCommandCountUpdatedHandler.java} | 39 +-
.../replication/ECContainerReplicaCount.java | 83 +-
.../replication/ECMisReplicationHandler.java | 71 +
.../replication/ECOverReplicationHandler.java | 43 +-
.../replication/ECUnderReplicationHandler.java | 307 ++-
.../replication/LegacyReplicationManager.java | 74 +-
.../replication/MisReplicationHandler.java | 190 ++
.../replication/OverReplicatedProcessor.java | 124 +-
.../RatisContainerReplicaCount.java | 99 +-
.../replication/RatisMisReplicationHandler.java | 78 +
.../replication/RatisOverReplicationHandler.java | 247 ++
.../replication/RatisUnderReplicationHandler.java | 242 ++
.../container/replication/ReplicationManager.java | 248 +-
.../replication/ReplicationManagerMetrics.java | 107 +-
.../replication/UnderReplicatedProcessor.java | 135 +-
.../replication/UnhealthyReplicationProcessor.java | 151 ++
.../ClosedWithMismatchedReplicasHandler.java | 64 +-
.../health/ClosedWithUnhealthyReplicasHandler.java | 129 +
.../health/ClosingContainerHandler.java | 12 +-
.../health/DeletingContainerHandler.java | 99 +
.../health/ECReplicationCheckHandler.java | 99 +-
.../replication/health/EmptyContainerHandler.java | 126 +
.../replication/health/OpenContainerHandler.java | 11 +-
.../health/QuasiClosedContainerHandler.java | 4 +
.../health/RatisReplicationCheckHandler.java | 219 ++
.../container/report/ContainerReportValidator.java | 104 +
.../hdds/scm/container/report}/package-info.java | 12 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 13 +-
.../hadoop/hdds/scm/ha/BackgroundSCMService.java | 2 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 7 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 5 +-
.../hdds/scm/metadata/SCMMetadataStoreMetrics.java | 11 -
.../apache/hadoop/hdds/scm/node/CommandQueue.java | 66 +-
.../hdds/scm/node/CommandQueueReportHandler.java | 3 +-
.../hadoop/hdds/scm/node/DatanodeAdminMonitor.java | 2 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 67 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 29 +-
.../hdds/scm/node/NodeDecommissionManager.java | 8 +-
.../hdds/scm/node/NodeDecommissionMetrics.java | 290 +++
.../apache/hadoop/hdds/scm/node/NodeManager.java | 19 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 36 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 42 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 37 +-
.../pipeline/PipelinePlacementPolicyFactory.java | 52 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 6 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 34 +-
.../scm/pipeline/WritableECContainerProvider.java | 11 +-
...inerLocationProtocolServerSideTranslatorPB.java | 17 +
.../hdds/scm/safemode/SCMSafeModeManager.java | 4 -
.../hdds/scm/server/ContainerReportQueue.java | 378 +++
.../hdds/scm/server/SCMClientProtocolServer.java | 18 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 86 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 41 +-
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 6 -
.../hdds/scm/server/StorageContainerManager.java | 273 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 102 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 431 +++-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 19 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 16 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 15 +-
.../scm/container/TestContainerManagerImpl.java | 12 +-
.../scm/container/TestContainerReportHandler.java | 59 +
.../TestIncrementalContainerReportHandler.java | 124 +-
.../container/balancer/TestContainerBalancer.java | 995 +-------
...alancer.java => TestContainerBalancerTask.java} | 228 +-
.../container/balancer/TestFindTargetStrategy.java | 34 +
.../algorithms/TestContainerPlacementFactory.java | 20 +-
.../TestContainerPlacementStatusDefault.java | 22 +-
.../TestSCMContainerPlacementRackScatter.java | 96 +-
.../container/replication/ReplicationTestUtil.java | 39 +-
.../TestContainerReplicaPendingOps.java | 127 +
.../TestDatanodeCommandCountUpdatedHandler.java | 49 +
.../replication/TestECContainerReplicaCount.java | 107 +
.../replication/TestECMisReplicationHandler.java | 174 ++
.../replication/TestECOverReplicationHandler.java | 44 +-
.../replication/TestECUnderReplicationHandler.java | 379 ++-
.../replication/TestLegacyReplicationManager.java | 37 +-
.../replication/TestMisReplicationHandler.java | 182 ++
.../replication/TestOverReplicatedProcessor.java | 41 +-
.../TestRatisContainerReplicaCount.java | 58 +-
.../TestRatisMisReplicationHandler.java | 178 ++
.../TestRatisOverReplicationHandler.java | 281 ++
.../TestRatisUnderReplicationHandler.java | 228 ++
.../replication/TestReplicationManager.java | 302 ++-
.../replication/TestReplicationManagerMetrics.java | 2 +
.../replication/TestUnderReplicatedProcessor.java | 65 +-
.../TestClosedWithMismatchedReplicasHandler.java | 16 +-
.../TestClosedWithUnhealthyReplicasHandler.java | 194 ++
.../health/TestClosingContainerHandler.java | 103 +-
.../health/TestDeletingContainerHandler.java | 238 ++
.../health/TestECReplicationCheckHandler.java | 378 ++-
...Handler.java => TestEmptyContainerHandler.java} | 177 +-
.../health/TestRatisReplicationCheckHandler.java | 545 ++++
.../report/TestContainerReportValidator.java | 77 +
.../hdds/scm/container/report/package-info.java | 19 +-
.../scm/metadata/TestSCMMetadataStoreImpl.java | 10 -
.../scm/node/DatanodeAdminMonitorTestUtil.java | 209 ++
.../hadoop/hdds/scm/node/TestCommandQueue.java | 101 +
.../scm/node/TestCommandQueueReportHandler.java | 30 +-
.../hdds/scm/node/TestContainerPlacement.java | 4 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 224 +-
.../hdds/scm/node/TestNodeDecommissionMetrics.java | 343 +++
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 74 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 67 +-
.../scm/pipeline/TestPipelinePlacementFactory.java | 215 ++
.../scm/pipeline/TestPipelinePlacementPolicy.java | 15 +-
.../scm/pipeline/TestRatisPipelineProvider.java | 22 +-
.../pipeline/TestWritableECContainerProvider.java | 5 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 8 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 27 +-
.../ozone/container/common/TestEndPoint.java | 8 +
.../testutils/ReplicationNodeManagerMock.java | 15 +-
hadoop-hdds/test-utils/pom.xml | 10 +-
.../org/apache/ozone/test/GenericTestUtils.java | 51 +
hadoop-hdds/tools/pom.xml | 8 +-
.../hdds/scm/cli/ContainerOperationClient.java | 169 +-
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 1 +
.../hadoop/hdds/scm/cli/cert/CleanExpired.java | 117 +
.../hadoop/hdds/scm/cli/cert/TestCleanExpired.java | 100 +
hadoop-ozone/client/pom.xml | 4 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 13 -
.../apache/hadoop/ozone/client/OzoneBucket.java | 58 +
.../org/apache/hadoop/ozone/client/OzoneKey.java | 40 +-
.../hadoop/ozone/client/OzoneKeyDetails.java | 11 +-
.../client/checksum/BaseFileChecksumHelper.java | 12 +-
.../client/checksum/ChecksumHelperFactory.java | 52 +
.../client/checksum/ECBlockChecksumComputer.java | 221 ++
.../client/checksum/ECFileChecksumHelper.java | 211 ++
.../client/io/BlockDataStreamOutputEntry.java | 287 +++
.../client/io/BlockDataStreamOutputEntryPool.java | 290 +++
.../ozone/client/io/BlockOutputStreamEntry.java | 13 +
.../ozone/client/io/ECBlockOutputStreamEntry.java | 34 +-
.../client/io/ECBlockOutputStreamEntryPool.java | 4 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 234 +-
...yOutputStream.java => KeyDataStreamOutput.java} | 334 +--
.../hadoop/ozone/client/io/KeyInputStream.java | 436 +---
.../hadoop/ozone/client/io/KeyOutputStream.java | 24 +-
.../client/io/MultipartCryptoKeyInputStream.java | 382 ---
.../ozone/client/io/OzoneCryptoInputStream.java | 157 +-
...utputStream.java => OzoneDataStreamOutput.java} | 49 +-
.../hadoop/ozone/client/io/OzoneOutputStream.java | 25 +-
.../ozone/client/protocol/ClientProtocol.java | 59 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 363 ++-
.../hadoop/ozone/client/MockOmTransport.java | 14 +
.../hadoop/ozone/client/TestOzoneECClient.java | 51 +-
hadoop-ozone/common/pom.xml | 4 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 12 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 13 +
.../ozone/om/helpers/KeyInfoWithVolumeContext.java | 110 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 70 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 55 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 106 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 71 +
.../ozone/om/protocolPB/OMAdminProtocolPB.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 32 +-
.../hadoop/ozone/om/helpers/TestOmBucketArgs.java | 91 +
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/datanode/pom.xml | 4 +-
hadoop-ozone/dev-support/checks/coverage.sh | 6 +-
hadoop-ozone/dev-support/intellij/log4j.properties | 2 +
.../dev-support/intellij/ozone-site-ha.xml | 174 ++
hadoop-ozone/dev-support/intellij/ozone-site.xml | 20 +
...torageContainerManager.xml => Datanode1-ha.xml} | 8 +-
.../{Datanode2.xml => Datanode2-ha.xml} | 4 +-
.../intellij/runConfigurations/Datanode2.xml | 2 +-
.../{Datanode2.xml => Datanode3-ha.xml} | 4 +-
.../intellij/runConfigurations/Datanode3.xml | 2 +-
...ageContainerManager.xml => OzoneFsShell-ha.xml} | 13 +-
...ageContainerManager.xml => OzoneManager-ha.xml} | 8 +-
...ontainerManager.xml => OzoneManagerInit-ha.xml} | 8 +-
...orageContainerManager.xml => OzoneShell-ha.xml} | 8 +-
...geContainerManager.xml => PrimordialSCM-ha.xml} | 4 +-
...ntainerManager.xml => PrimordialSCMInit-ha.xml} | 4 +-
.../{StorageContainerManager.xml => Recon-ha.xml} | 8 +-
.../{StorageContainerManager.xml => Scm2-ha.xml} | 4 +-
...geContainerManager.xml => Scm2Bootstrap-ha.xml} | 4 +-
.../{StorageContainerManager.xml => Scm3-ha.xml} | 4 +-
...geContainerManager.xml => Scm3Bootstrap-ha.xml} | 4 +-
.../{StorageContainerManager.xml => ScmRoles.xml} | 8 +-
.../runConfigurations/StorageContainerManager.xml | 2 +-
.../dist/dev-support/bin/dist-layout-stitching | 1 -
hadoop-ozone/dist/pom.xml | 16 +-
.../{ozone-csi => ozone-legacy-bucket}/.env | 0
.../src/main/compose/ozone-legacy-bucket/README.md | 18 +-
.../docker-compose.yaml | 0
.../{ozone => ozone-legacy-bucket}/docker-config | 2 +
.../compose/ozone-legacy-bucket/test.sh} | 29 +-
.../main/compose/ozone-om-ha/docker-compose.yaml | 5 +
.../dist/src/main/compose/ozone/docker-config | 2 +
.../src/main/compose/ozonesecure/docker-config | 2 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 146 +-
hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +-
.../main/license/bin/licenses/LICENSE-angular.txt | 2 +-
.../src/main/license/bin/licenses/LICENSE-d3.txt | 2 +-
.../licenses/LICENSE-glyphicons.txt} | 4 +-
.../main/license/bin/licenses/LICENSE-guava.txt | 13 +
.../LICENSE-javax.activation-activation.txt | 3 -
.../LICENSE-net.sf.jopt-simple-jopt-simple.txt | 24 -
.../license/bin/licenses/LICENSE-org.aspectj.html | 91 -
.../license/bin/licenses/LICENSE-org.aspectj.txt | 279 ++
.../bin/licenses/LICENSE-org.ow2.asm-asm.txt | 1 -
.../license/bin/licenses/LICENSE-org.slf4j.txt | 2 +-
...y-misc.txt => NOTICE-ratis-thirdparty-misc.txt} | 24 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 14 +-
.../main/license/src/licenses/LICENSE-angular.txt | 2 +-
.../{LICENSE-angular.txt => LICENSE-bootstrap.txt} | 4 +-
.../src/main/license/src/licenses/LICENSE-d3.txt | 4 +-
...{LICENSE-angular.txt => LICENSE-glyphicons.txt} | 4 +-
.../main/license/src/licenses/LICENSE-guava.txt | 13 +
.../main/license/src/licenses/LICENSE-jquery.txt | 2 +-
.../src/main/license/src/licenses/LICENSE-nvd3.txt | 17 +-
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 7 +-
.../smoketest/compatibility/dn-one-rocksdb.robot | 2 +-
.../dist/src/main/smoketest/ec/ozonefs.robot | 61 +
.../src/main/smoketest/freon/read-write-key.robot | 53 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 37 +-
...n-fso-nssummary.robot => recon-nssummary.robot} | 12 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 5 +
.../dist/src/main/smoketest/s3/objectputget.robot | 26 +
hadoop-ozone/dist/src/shell/ozone/ozone | 24 +-
.../dist/src/shell/ozone/ozone-functions.sh | 87 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 -
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 4 +-
hadoop-ozone/integration-test/pom.xml | 4 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 111 +-
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 160 ++
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 177 ++
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 182 +-
.../fs/ozone/TestOzoneFileSystemWithStreaming.java | 158 ++
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 2 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 158 +-
.../hdds/scm/TestSCMDatanodeProtocolServer.java | 54 +
.../TestContainerStateManagerIntegration.java | 2 +-
.../TestRatisPipelineCreateAndDestroy.java | 9 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 17 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 20 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 23 +
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 4 +
.../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 262 +-
.../hadoop/ozone/TestStorageContainerManager.java | 157 +-
.../ozone/TestStorageContainerManagerHelper.java | 20 +-
.../ozone/client/CertificateClientTestImpl.java | 164 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 3 -
.../client/rpc/TestBlockDataStreamOutput.java | 273 ++
.../ozone/client/rpc/TestBlockOutputStream.java | 2 -
.../rpc/TestBlockOutputStreamFlushDelay.java | 2 -
.../rpc/TestBlockOutputStreamWithFailures.java | 2 -
...estBlockOutputStreamWithFailuresFlushDelay.java | 2 -
.../rpc/TestCloseContainerHandlingByClient.java | 9 -
.../client/rpc/TestContainerStateMachine.java | 2 -
.../TestContainerStateMachineFailureOnRead.java | 2 -
.../rpc/TestContainerStateMachineFailures.java | 1 -
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../rpc/TestContainerStateMachineStream.java | 219 ++
.../client/rpc/TestDeleteWithSlowFollower.java | 3 -
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 24 +-
.../client/rpc/TestFailureHandlingByClient.java | 5 -
.../rpc/TestFailureHandlingByClientFlushDelay.java | 1 -
.../rpc/TestMultiBlockWritesWithDnFailures.java | 4 -
.../client/rpc/TestOzoneAtRestEncryption.java | 7 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 441 ++--
...estOzoneClientRetriesOnExceptionFlushDelay.java | 4 -
.../rpc/TestOzoneClientRetriesOnExceptions.java | 3 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 106 +-
.../client/rpc/TestOzoneRpcClientWithRatis.java | 131 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 2 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 1 -
.../client/rpc/TestValidateBCSIDOnRestart.java | 2 +-
.../client/rpc/read/TestChunkInputStream.java | 6 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 -
.../ozone/client/rpc/read/TestKeyInputStream.java | 6 +-
.../ozone/container/TestECContainerRecovery.java | 130 +-
.../apache/hadoop/ozone/container/TestHelper.java | 42 +-
.../commandhandler/TestBlockDeletion.java | 10 +-
.../TestCloseContainerByPipeline.java | 6 +-
.../commandhandler/TestCloseContainerHandler.java | 1 -
.../commandhandler/TestDeleteContainerHandler.java | 1 -
.../metrics/TestDatanodeQueueMetrics.java | 111 +
.../ozoneimpl/TestOzoneContainerWithTLS.java | 239 +-
.../container/server/TestContainerServer.java | 3 +-
.../server/TestSecureContainerServer.java | 4 +
.../ozone/dn/ratis/TestDnRatisLogParser.java | 1 -
.../hadoop/ozone/dn/scanner/TestDataScanner.java | 182 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 36 +-
.../ozone/om/TestContainerReportWithKeys.java | 1 -
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 80 +-
.../org/apache/hadoop/ozone/om/TestKeyPurging.java | 1 +
.../org/apache/hadoop/ozone/om/TestLDBCli.java | 21 +
.../hadoop/ozone/om/TestOMEpochForNonRatis.java | 2 +-
.../ozone/om/TestObjectStoreWithLegacyFS.java | 52 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 171 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 2 -
.../ozone/om/TestOmContainerLocationCache.java | 674 +++++
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 74 +-
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 11 +-
.../ozone/om/TestOzoneManagerListVolumes.java | 48 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 3 -
.../hadoop/ozone/om/TestSecureOzoneManager.java | 20 +-
.../ozone/om/multitenant/RangerUserRequest.java | 3 +-
.../TestRangerBGSyncService.java | 7 +-
.../ozone/parser/TestOzoneHARatisLogParser.java | 61 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 4 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 40 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 5 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 20 -
.../ozone/scm/TestStorageContainerManagerHA.java | 2 +-
.../TestSCMPipelineBytesWrittenMetrics.java | 2 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 6 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 41 +-
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 2 +-
hadoop-ozone/interface-client/pom.xml | 4 +-
.../src/main/proto/OmClientProtocol.proto | 35 +-
.../interface-client/src/main/resources/proto.lock | 2672 ++++++++++++++++----
hadoop-ozone/interface-storage/pom.xml | 4 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 12 +-
hadoop-ozone/ozone-manager/pom.xml | 5 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 80 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 11 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 231 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 33 +
.../hadoop/ozone/om/OMMultiTenantManager.java | 2 +-
.../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 2 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 71 +-
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 4 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 17 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 83 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 35 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 452 ++--
.../hadoop/ozone/om/OzonePrefixPathImpl.java | 2 -
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 8 +
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 4 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 105 +-
.../org/apache/hadoop/ozone/om/VolumeManager.java | 2 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 33 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 27 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 8 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 -
.../om/request/bucket/OMBucketCreateRequest.java | 17 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 48 +-
.../om/request/file/OMDirectoryCreateRequest.java | 10 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 25 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 11 +-
.../request/file/OMFileCreateRequestWithFSO.java | 6 +-
.../om/request/key/OMAllocateBlockRequest.java | 6 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 3 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 80 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 31 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 29 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 25 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 6 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 69 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../S3MultipartUploadCommitPartRequest.java | 9 +-
.../S3MultipartUploadCompleteRequest.java | 3 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 3 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 11 +
.../response/file/OMDirectoryCreateResponse.java | 9 +-
.../file/OMDirectoryCreateResponseWithFSO.java | 10 +-
.../response/file/OMFileCreateResponseWithFSO.java | 6 +
.../key/OMDirectoriesPurgeResponseWithFSO.java | 19 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 6 +
.../ozone/om/response/key/OMKeyCreateResponse.java | 7 +
.../response/key/OMKeyDeleteResponseWithFSO.java | 4 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 4 +-
.../S3InitiateMultipartUploadResponseWithFSO.java | 12 +-
.../om/{ => service}/DirectoryDeletingService.java | 280 +-
.../ozone/om/{ => service}/KeyDeletingService.java | 33 +-
.../OMRangerBGSyncService.java | 8 +-
.../om/{ => service}/OpenKeyCleanupService.java | 5 +-
.../hadoop/ozone/om/service/package-info.java} | 9 +-
.../protocolPB/OzoneManagerRequestHandler.java | 76 +-
.../hadoop/ozone/security/AWSV4AuthValidator.java | 4 +-
.../hadoop/ozone/security/OMCertificateClient.java | 203 ++
.../webapps/ozoneManager/om-overview.html | 4 +
.../apache/hadoop/ozone/om/TestChunkStreams.java | 150 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 179 +-
.../hadoop/ozone/om/TestSstFilteringService.java | 65 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../ozone/om/request/OMRequestTestUtils.java | 15 +
.../ozone/om/request/bucket/TestBucketRequest.java | 1 +
.../request/bucket/TestOMBucketCreateRequest.java | 46 +
.../bucket/TestOMBucketCreateRequestWithFSO.java | 71 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 336 ++-
.../request/file/TestOMDirectoryCreateRequest.java | 37 +
.../file/TestOMDirectoryCreateRequestWithFSO.java | 39 +
.../om/request/file/TestOMFileCreateRequest.java | 57 +-
.../file/TestOMFileCreateRequestWithFSO.java | 31 +
.../om/request/key/TestOMAllocateBlockRequest.java | 24 +
.../TestOMDirectoriesPurgeRequestAndResponse.java | 297 +++
.../om/request/key/TestOMKeyCommitRequest.java | 127 +-
.../om/request/key/TestOMKeyCreateRequest.java | 55 +
.../request/key/TestOMKeyCreateRequestWithFSO.java | 6 +
.../TestS3MultipartUploadCompleteRequest.java | 6 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 6 +
.../file/TestOMDirectoryCreateResponse.java | 19 +-
.../file/TestOMDirectoryCreateResponseWithFSO.java | 19 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 7 +-
.../s3/multipart/TestS3MultipartResponse.java | 12 +-
.../TestS3MultipartUploadAbortResponse.java | 3 +-
.../TestS3MultipartUploadAbortResponseWithFSO.java | 12 +-
.../om/{ => service}/TestKeyDeletingService.java | 177 +-
.../{ => service}/TestOpenKeyCleanupService.java | 6 +-
.../security/TestOmCertificateClientInit.java | 67 +-
.../TestOzoneDelegationTokenSecretManager.java | 1 -
hadoop-ozone/ozonefs-common/pom.xml | 8 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 37 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 74 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 61 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 160 +-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 57 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 +
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 49 +-
.../hadoop/fs/ozone/OzoneFSDataStreamOutput.java | 103 +
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 19 +-
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 32 -
hadoop-ozone/ozonefs-hadoop2/pom.xml | 10 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 20 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 20 +-
.../pom.xml | 104 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
hadoop-ozone/pom.xml | 12 +-
hadoop-ozone/recon-codegen/pom.xml | 3 +-
hadoop-ozone/recon/pom.xml | 4 +-
.../hadoop/ozone/recon/ReconControllerModule.java | 4 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 137 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 20 +-
.../ozone/recon/api/ClusterStateEndpoint.java | 23 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 1 +
.../ozone/recon/api/TriggerDBSyncEndpoint.java | 52 +
.../ozone/recon/api/handlers/BucketHandler.java | 56 +-
.../recon/api/handlers/DirectoryEntityHandler.java | 20 +-
.../ozone/recon/api/handlers/FSOBucketHandler.java | 4 +-
.../ozone/recon/api/handlers/KeyEntityHandler.java | 3 +-
.../recon/api/handlers/LegacyBucketHandler.java | 325 +++
.../recon/api/handlers/RootEntityHandler.java | 24 +-
.../recon/api/types/ClusterStateResponse.java | 36 +
.../hadoop/ozone/recon/api/types/NSSummary.java | 4 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 67 +-
.../ozone/recon/fsck/ReconSafeModeMgrTask.java | 122 +
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 33 +-
.../ozone/recon/scm/ReconContainerManager.java | 27 +-
.../recon/scm/ReconDatanodeProtocolServer.java | 2 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 12 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../ozone/recon/scm/ReconSafeModeManager.java | 12 +-
.../ozone/recon/scm/ReconStaleNodeHandler.java | 58 +
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 4 +
.../scm/ReconStorageContainerManagerFacade.java | 250 +-
.../recon/spi/OzoneManagerServiceProvider.java | 6 +
.../recon/spi/StorageContainerServiceProvider.java | 21 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 153 +-
.../impl/ReconContainerMetadataManagerImpl.java | 34 +-
.../impl/StorageContainerServiceProviderImpl.java | 18 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 233 +-
...yTask.java => NSSummaryTaskDbEventHandler.java} | 95 +-
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 71 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 310 +++
.../hadoop/ozone/recon/tasks/ReconTaskConfig.java | 17 +
.../resources/webapps/recon/ozone-recon-web/NOTICE | 5 -
.../webapps/recon/ozone-recon-web/api/db.json | 462 +++-
.../webapps/recon/ozone-recon-web/api/routes.json | 4 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 18 +-
.../src/components/navBar/navBar.tsx | 5 +
.../ozone-recon-web/src/utils/autoReloadHelper.tsx | 1 +
.../recon/ozone-recon-web/src/utils/themeIcons.tsx | 6 +-
.../src/views/datanodes/datanodes.tsx | 20 +-
.../src/views/diskUsage/diskUsage.less | 36 +-
.../src/views/diskUsage/diskUsage.tsx | 68 +-
.../src/views/insights/insights.tsx | 10 +-
.../views/missingContainers/missingContainers.tsx | 240 +-
.../src/views/overview/overview.tsx | 49 +-
.../src/views/pipelines/pipelines.tsx | 4 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 28 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 8 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 114 +-
...O.java => TestNSSummaryEndpointWithLegacy.java} | 389 +--
.../recon/api/TestTotalOpenContainerCount.java | 396 +++
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 143 ++
.../ozone/recon/api/filters/TestAdminFilter.java | 2 +
.../ozone/recon/fsck/TestContainerHealthTask.java | 17 +-
.../TestContainerHealthTaskRecordGenerator.java | 42 +-
.../scm/AbstractReconContainerManagerTest.java | 4 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 70 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 492 ++++
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 21 +-
...thFSO.java => TestNSSummaryTaskWithLegacy.java} | 328 ++-
.../ozone/recon/tasks/TestTableCountTask.java | 2 +-
hadoop-ozone/s3gateway/pom.xml | 5 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 72 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 31 +-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 2 +
.../hadoop/ozone/client/ClientProtocolStub.java | 37 +
.../hadoop/ozone/client/OzoneBucketStub.java | 4 +-
.../hadoop/ozone/client/OzoneOutputStreamStub.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 106 +
.../ozone/s3/endpoint/TestPermissionCheck.java | 3 +-
hadoop-ozone/tools/pom.xml | 4 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 7 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 7 +-
.../ozone/admin/nssummary/NSSummaryAdmin.java | 57 +
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 9 +-
.../admin/nssummary/QuotaUsageSubCommand.java | 7 +-
.../ozone/admin/nssummary/SummarySubCommand.java | 7 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 1 -
.../org/apache/hadoop/ozone/debug/DBScanner.java | 64 +-
.../ozone/debug/container/ExportSubcommand.java | 4 +-
.../ozone/freon/ClosedContainerReplicator.java | 14 +-
.../hadoop/ozone/freon/ContentGenerator.java | 18 +
.../java/org/apache/hadoop/ozone/freon/Freon.java | 2 +
.../hadoop/ozone/freon/KeyGeneratorUtil.java | 50 +
.../ozone/freon/OzoneClientKeyGenerator.java | 29 +-
.../ozone/freon/OzoneClientKeyReadWriteOps.java | 245 ++
.../hadoop/ozone/freon/RangeKeysGenerator.java | 164 ++
.../java/org/apache/hadoop/ozone/shell/Shell.java | 21 +-
.../ozone/shell/bucket/ClearQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/bucket/SetQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/keys/GetKeyHandler.java | 4 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 67 +-
.../ozone/shell/volume/ClearQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/volume/SetQuotaHandler.java | 11 +-
pom.xml | 442 +---
843 files changed, 43921 insertions(+), 13330 deletions(-)
copy .github/workflows/{post-commit.yml => ci.yml} (89%)
copy .github/workflows/{close-pending.yaml => scheduled_ci.yml} (68%)
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/AbstractDataStreamOutput.java
copy
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/{BlockOutputStream.java
=> BlockDataStreamOutput.java} (52%)
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ByteBufferStreamOutput.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/MultipartInputStream.java
copy
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/{BlockExtendedInputStream.java
=> PartInputStream.java} (69%)
copy
hadoop-hdds/{container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestContainerLayoutVersion.java
=> client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBuffer.java}
(51%)
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamCommitWatcher.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CacheMetrics.java
copy
hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CheckedRunnable.java
(52%)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/util/CheckedSupplier.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MetricUtil.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/AuditLogTestUtils.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeQueueMetrics.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/LocalStream.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionMetrics.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyValueStreamDataChannel.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/StreamDataChannelBase.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerScanner.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandScannerMetrics.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/CopyContainerCompression.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerCommandHandler.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestKeyValueStreamDataChannel.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
create mode 100644 hadoop-hdds/docs/content/feature/Streaming-Write-Pipeline.md
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/KeyStoresFactory.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/MonitoringTimerTask.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/ReloadingX509KeyManager.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/ReloadingX509TrustManager.java
copy
hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/package-info.java
(58%)
delete mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java
copy
hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IEventInfo.java
(80%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DecayRpcSchedulerUtil.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetricsUtil.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/PrometheusMetricsSinkUtil.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TableCacheMetrics.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/UgiMetricsUtil.java
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSafeModeManager.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/CacheStats.java
(58%)
rename
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMEchoRPCRequest.java
=>
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/CacheStatsRecorder.java
(50%)
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/TestPemFileBasedKeyStoresFactory.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/TestReloadingX509KeyManager.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/TestReloadingX509TrustManager.java
copy
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
=>
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java
(71%)
copy
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/{TestCertificateClientInit.java
=> TestDnCertificateClientInit.java} (70%)
rename
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/{TestPrometheusMetricsSink.java
=> TestPrometheusMetricsIntegration.java} (55%)
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestDecayRpcSchedulerUtil.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestPrometheusMetricsSinkUtil.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestUgiMetricsUtil.java
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/{ContainerBalancer.java
=> ContainerBalancerTask.java} (64%)
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/OzoneManagerServiceProvider.java
=>
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaPendingOpsSubscriber.java
(55%)
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/{node/CommandQueueReportHandler.java
=> container/replication/DatanodeCommandCountUpdatedHandler.java} (50%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECMisReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MisReplicationHandler.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ =>
replication}/RatisContainerReplicaCount.java (73%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisMisReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisOverReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/RatisUnderReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/UnhealthyReplicationProcessor.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosedWithUnhealthyReplicasHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/DeletingContainerHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/EmptyContainerHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/RatisReplicationCheckHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/report/ContainerReportValidator.java
rename
hadoop-hdds/{erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder
=>
server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/report}/package-info.java
(73%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionMetrics.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelinePlacementPolicyFactory.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ContainerReportQueue.java
copy
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/{TestContainerBalancer.java
=> TestContainerBalancerTask.java} (87%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestDatanodeCommandCountUpdatedHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestECMisReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestMisReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisMisReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisOverReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestRatisUnderReplicationHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosedWithUnhealthyReplicasHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestDeletingContainerHandler.java
copy
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/{TestClosingContainerHandler.java
=> TestEmptyContainerHandler.java} (51%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestRatisReplicationCheckHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/report/TestContainerReportValidator.java
copy
hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
=>
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/report/package-info.java
(71%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorTestUtil.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestCommandQueue.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionMetrics.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelinePlacementFactory.java
create mode 100644
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/CleanExpired.java
create mode 100644
hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/cert/TestCleanExpired.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ChecksumHelperFactory.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECFileChecksumHelper.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntry.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
copy
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/{KeyOutputStream.java
=> KeyDataStreamOutput.java} (60%)
delete mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/MultipartCryptoKeyInputStream.java
copy
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/{OzoneOutputStream.java
=> OzoneDataStreamOutput.java} (52%)
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyInfoWithVolumeContext.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketArgs.java
create mode 100644 hadoop-ozone/dev-support/intellij/ozone-site-ha.xml
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Datanode1-ha.xml} (78%)
copy hadoop-ozone/dev-support/intellij/runConfigurations/{Datanode2.xml =>
Datanode2-ha.xml} (75%)
copy hadoop-ozone/dev-support/intellij/runConfigurations/{Datanode2.xml =>
Datanode3-ha.xml} (75%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> OzoneFsShell-ha.xml} (70%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> OzoneManager-ha.xml} (80%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> OzoneManagerInit-ha.xml} (80%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> OzoneShell-ha.xml} (80%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> PrimordialSCM-ha.xml} (85%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> PrimordialSCMInit-ha.xml} (83%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Recon-ha.xml} (80%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Scm2-ha.xml} (84%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Scm2Bootstrap-ha.xml} (83%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Scm3-ha.xml} (84%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> Scm3Bootstrap-ha.xml} (83%)
copy
hadoop-ozone/dev-support/intellij/runConfigurations/{StorageContainerManager.xml
=> ScmRoles.xml} (84%)
copy hadoop-ozone/dist/src/main/compose/{ozone-csi =>
ozone-legacy-bucket}/.env (100%)
copy hadoop-hdds/dev-support/checkstyle/suppressions.xml =>
hadoop-ozone/dist/src/main/compose/ozone-legacy-bucket/README.md (59%)
copy hadoop-ozone/dist/src/main/compose/{ozone =>
ozone-legacy-bucket}/docker-compose.yaml (100%)
copy hadoop-ozone/dist/src/main/compose/{ozone =>
ozone-legacy-bucket}/docker-config (95%)
rename hadoop-ozone/dist/src/{shell/conf/ozone-shell-log4j.properties =>
main/compose/ozone-legacy-bucket/test.sh} (51%)
copy hadoop-ozone/dist/src/main/license/{src/licenses/LICENSE-angular.txt =>
bin/licenses/LICENSE-glyphicons.txt} (93%)
create mode 100644
hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-guava.txt
delete mode 100644
hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-net.sf.jopt-simple-jopt-simple.txt
delete mode 100644
hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.aspectj.html
create mode 100644
hadoop-ozone/dist/src/main/license/bin/licenses/LICENSE-org.aspectj.txt
rename
hadoop-ozone/dist/src/main/license/bin/licenses/{NOTICE-ratis-thirtparty-misc.txt
=> NOTICE-ratis-thirdparty-misc.txt} (94%)
copy hadoop-ozone/dist/src/main/license/src/licenses/{LICENSE-angular.txt =>
LICENSE-bootstrap.txt} (93%)
copy hadoop-ozone/dist/src/main/license/src/licenses/{LICENSE-angular.txt =>
LICENSE-glyphicons.txt} (93%)
create mode 100644
hadoop-ozone/dist/src/main/license/src/licenses/LICENSE-guava.txt
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/ozonefs.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot
rename hadoop-ozone/dist/src/main/smoketest/recon/{recon-fso-nssummary.robot
=> recon-nssummary.robot} (92%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDatanodeProtocolServer.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestDatanodeQueueMetrics.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmContainerLocationCache.java
rename
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/{multitenant
=> service}/TestRangerBGSyncService.java (98%)
rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{
=> service}/DirectoryDeletingService.java (50%)
rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{
=> service}/KeyDeletingService.java (92%)
rename
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{multitenant
=> service}/OMRangerBGSyncService.java (98%)
rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/{
=> service}/OpenKeyCleanupService.java (97%)
copy
hadoop-ozone/{ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
=>
ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/package-info.java}
(82%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OMCertificateClient.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
rename hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/{
=> service}/TestKeyDeletingService.java (66%)
rename hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/{
=> service}/TestOpenKeyCleanupService.java (97%)
rename
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestCertificateClientInit.java
=>
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOmCertificateClientInit.java
(72%)
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSDataStreamOutput.java
copy hadoop-ozone/{ozonefs-hadoop2 => ozonefs-hadoop3-client}/pom.xml (51%)
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/TriggerDBSyncEndpoint.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/handlers/LegacyBucketHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconSafeModeMgrTask.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStaleNodeHandler.java
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/{NSSummaryTask.java
=> NSSummaryTaskDbEventHandler.java} (73%)
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java
delete mode 100644
hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/NOTICE
copy
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/{TestNSSummaryEndpointWithFSO.java
=> TestNSSummaryEndpointWithLegacy.java} (83%)
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTotalOpenContainerCount.java
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTriggerDBSyncEndpoint.java
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTask.java
copy
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/{TestNSSummaryTaskWithFSO.java
=> TestNSSummaryTaskWithLegacy.java} (68%)
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestEndpointBase.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
