This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a change to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git
omit c17be7013a HDDS-7478. [Ozone-Streaming] NPE in when creating a file
with o3fs. (#3949)
omit 7fc85ee1fe HDDS-7425. Add documentation for the new Streaming Pipeline
feature. (#3913)
omit cfdf9cd171 HDDS-7438. [Ozone-Streaming] Add a createStreamKey method
to OzoneBucket. (#3914)
omit 53af9086bb HDDS-7431. [Ozone-Streaming] Disable data steam by default.
(#3900)
omit 6319ce8ad6 HDDS-6955. [Ozone-streaming] Add explicit stream flag in
ozone shell (#3559)
omit b3a6cfccc0 HDDS-6867. [Ozone-Streaming] PutKeyHandler should not use
streaming to put EC key. (#3516)
omit 317053dbcc HDDS-6842. [Ozone-Streaming] Reduce the number of watch
requests in StreamCommitWatcher. (#3492)
omit a7eb40e6fd HDDS-6592. [Ozone-Streaming] Fix
ContainerStateMachine#applyTransaction assert error (#3315)
omit d1b7a92221 HDDS-5666. Add option to createKey via streaming api in
Freon (#2574)
omit d0407c1ea6 HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at
the end of the stream. (#3229)
omit 06bc9689ca HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput.
(#3195)
omit b42829cfc3 HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters.
(#3207)
omit 1f097e1c46 HDDS-6388. [Ozone-Streaming] Streaming write support both
pipeline model and star model (#3145)
omit 328c389b0b HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
omit 4ce474c12a HDDS-6229. [Ozone-Streaming] Data Channel abstraction on
datanode (#3023)
omit fce0dfad3a HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE
(#3060)
omit 9c6904ddfd HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support
FlushDelay. (#3002)
omit 20f1682834 HDDS-6298. Add XceiverServerRatis stream config (#3070)
omit 340f9594e8 HDDS-6138.[Ozone-Streaming] Define a limit on the size of
the retry bufferList. (#2946)
omit dd626842f7 HDDS-6281. Update ratis version to 2.3.0-94db58b-SNAPSHOT
version (#3059)
omit ddf9f84df0 HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
omit 156cd46ad6 HDDS-6139. [Ozone-Streaming] Fix incorrect computation of
totalAckDataLength. (#2978)
omit 58da69f0aa HDDS-6130. [Ozone-Streaming] When releaseBuffers will get
“Couldn 't find the required future” (#2939)
omit 28fdecd856 HDDS-6039. Define a minimum packet size during streaming
writes. (#2883)
omit 9d5ccb3067 HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer
fixed boundary for streaming writes. (#2866)
omit d81ca2ab63 HDDS-5743. [Ozone-Streaming] Add option to write files via
streaming api in ofs and o3fs. (#2770)
omit 2054fe4dad HDDS-5879. [Ozone-Streaming] OzoneBucket add the
createMultipartStreamKey method (#2760)
omit 7290ff9827 HDDS-5961. [Ozone-Streaming] update the usage space of
Containers in the stream write (#2833)
omit 029055b113 HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream
config (#2841)
omit e4a0f00325 HDDS-5763. Provide an Executor for each LocalStream in
ContainerStateMachine (#2782)
omit 483eae8883 HDDS-5895. [Ozone-Streaming] Make
raft.server.data-stream.client.pool.size configurable (#2766)
omit 39587086ef HDDS-5674.[Ozone-Streaming] Handle client retries on
exception (#2701)
omit a6968d57f2 HDDS-5849. [Ozone-Streaming]Write exceptions occur after
checksum is enabled (#2729)
omit 8124a1cb41 HDDS-5486. [Ozone-Streaming] Streaming supports writing in
Pipline mode (#2682)
omit 72bc75006b HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
omit 8853c8a72b HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to
ByteBufferStreamOutput (#2603)
omit db89077fe4 HDDS-5599. [Ozone-Streaming]drop BufferPool and
ChunkBuffer to avoid buffer copying (#2557)
omit e6be97e0c6 HDDS-5488. [Ozone-Streaming] Add a new
BlockOutputStream/KeyOutputStream to support streaming api (#2495)
omit c69955ed74 HDDS-5480. [Ozone-Streaming] Client and server should
support stream setup. (#2452)
omit 28d30359e4 HDDS-5481. Fix stream() and link() method in
ContainerStateMachine. (#2451)
omit 92a8081d73 HDDS-5452. Add link method to ContainerStateMachine for
Ratis streaming (#2422)
omit d4bf89af1b HDDS-5366. [Ozone-Streaming] Implement stream method to
ContainerStateMachine. (#2358). Contributed by mingchao zhao
add 715a3f6d06 Upgrade RocksDB to 7.7.3 (#3924)
add f95da6c66e HDDS-7451. Add UT of SyncDataFromOM for
SequenceNumberNotFoundException (#3927)
add 14a1668521 HDDS-7326. Intermittent timeout in
TestECContainerRecovery.testContainerRecoveryOverReplicationProcessing (#3941)
add a499123287 HDDS-6896. EC: ReplicationManager - Add relevant metrics to
the various ReplicationManager classes (#3909)
add f9df0593e4 HDDS-5112. Recon Insights page does not list buckets when
only one volume is present. (#3907)
add 436b4b21eb HDDS-2642. Expose decommission / maintenance metrics via
JMX (#3781)
add 39dfe7ad86 HDDS-7453. Check certificate expiration at service startup,
renew if necessary. (#3930)
add 0af369e25f HDDS-7440. Delete package-info class for
org.apache.hadoop.io.erasurecode.rawcoder to avoid conflict with Hadoop (#3925)
add 3d3e6cb19a HDDS-7314. Recon graceful shutdown while closing RocksDB
handle. (#3931)
add 9a25a8ea7e HDDS-7360. Investigate TestRootedOzoneFileSystem test
cases. (#3928)
add 39747954a6 HDDS-7323. Recon: Auto refresh toggle is switched back when
visiting new site (#3868)
add fa5695b677 HDDS-7456 Recon: Refreshed at not working on Pipeline and
DataNode page (#3933)
add 03fa68109a HDDS-7408. Recon: Refactor container status page (#3916)
add ec0d24a9da HDDS-4385. Search function using container ID on Recon
Missing Container (#3917)
add 4dc3083954 HDDS-7469. Recon: Add more information to Open Containers
column on on the Datanodes page. (#3944)
add c4e9e348fa HDDS-7446. Cleanup possibly onDisk unreferenced blocks
(#3923)
add a13c62b605 HDDS-7471. EC: Notify ReplicationManager when a heartbeat
updates datanode command counts (#3945)
add a8a0a4533c HDDS-7475. Remove outdated Hadoop dependencies (#3950)
add b3e7f3c320 HDDS-7485. Bump commons-codec to 1.15 (#3955)
add 73d9082168 HDDS-7468. Recon: clusterState API Changes for adding
missingContainerCount (#3952)
add 0f2f15b79f HDDS-7490. fix metaTable error (#3961)
add b46f961b59 HDDS-7255. Add metrics for container reports events (#3936)
add 006df10d67 HDDS-7101. EC: ReplicationManager - handle UNHEALTHY
replicas (#3962)
add 8518ed1cff HDDS-6971. EC: Add EC block checksum computer. (#3790)
add 95568ab735 HDDS-7095. allow on demand scanning for containers (#3788)
add ba41ef5041 HDDS-7487. Support fine grained certificate lifetime for
efficient test. (#3954)
add 7e38bfe164 HDDS-7488. EC: ReplicationManager: Move Mis-Replicated into
a separate unhealthy state (#3956)
add 559ad868ef HDDS-4402. Recon dashboard page does not load until missing
containers API returns data. (#3968)
add 412a13aa56 HDDS-7245. Recon: Add button to trigger OM DB Sync
(Frontend) (#3929)
add 1bc9c8a0ac HDDS-7474. Recon: Add count to containers page tab label
(#3951)
add d476389a4b HDDS-7260. dfs -du should return correct disk space
consumed by all replicas (#3810)
add 6ff33075f8 HDDS-7470. [Quota] bucket is created crossing quota makes
OM crash and unable to start (#3960)
add 6a0d4d5c7f HDDS-7346. Cannot set bucket args when the bucket has quota
set (#3975)
add 85e7cd1867 HDDS-7498. Add permission check when `--user` is specified
in `ozone sh volume list` (#3971)
add 5e23afee10 HDDS-4565. [FSO]Delete : quotaReleased on directory
deletion should consider all sub paths (#3973)
add edefe79434 HDDS-7476. SCM root CA stored under hdds.datanode.dir if
ozone.metadata.dirs is not specified (#3948)
add 0ed0240c66 HDDS-7280. New test addition to
TestOMBucketCreateRequestWithFSO (#3794)
add 0f0645ed23 HDDS-5526. ContainerBalancer#checkConditionsForBalancing
pre-emptively checks iteration limits. (#3895)
add 99493d52f8 HDDS-7518. Fix intermittent failure in ozonesecure
replication test (#3986)
add 1e86860e1b HDDS-7462. EC: Fix Reconstruction Issue with
StaleRecoveringContainerScrubbingService (#3939)
add cb9b5d06dd HDDS-7092. EC: Offline Recovery with simultaneous Over
Replication & Under Replication (#3984)
add c8648b159e HDDS-7528. EC: ReplicationManager - refactor logic to send
datanode commands into a central place (#3990)
add 1370de18b8 HDDS-7532. EC: ReplicationManager - remove calls to
ECHealthCheck from under and over replication processing (#3995)
add 864397d1c6 HDDS-7187. EC: Retry failed writes before rewrite to a new
block group (#3977)
add b92f7c58f5 HDDS-7481. Set quota with reduced size than actual used do
not fail (#3978)
add 302097fabb HDDS-7353. Cleanup of unused code in SCM (#3905)
add 6abee8a48c HDDS-7534. Intermittent failure in
TestKeyDeletingService#checkDeletionForEmptyKey (#3998)
add 7f3f28c54e HDDS-7538. Remove unused OmUtils#getMD5Digest (#3996)
add 4145cf5a08 HDDS-7449. Avoid overwriting bucket encryption properties
when quota/replication config are set on an existing bucket (#3992)
add 6570c19081 HDDS-7535. Eliminate duplicated config in
LegacyReplicationManager (#4000)
add 7af8d06d87 HDDS-7289. Bump protobuf-java from 3.19.2 to 3.19.6 (#4003)
add 27ff3b6442 HDDS-7486. Support KeyStoreFactory which supports
keyManager and trustManager reload (#3959)
add e50c443d24 HDDS-7536. Remove unused dependency declarations (#4001)
add 8db02c864b HDDS-7500. Ozone recon server Memory Optimization. (#3974)
add f552cf21a1 HDDS-7540. Separate scheduled CI from push/PR workflows
(#4004)
add 829dfce410 HDDS-7554. Recon UI should show DORMANT in pipeline status
filter (#4010)
add eeeb0b548c HDDS-6615. EC: Improve write performance by pipelining
encode and flush (#3994)
add f1d651d0b5 HDDS-7561. Improve setquota, clrquota CLI usage (#4016)
add 627c90c233 HDDS-7510. Recon: Return number of open containers in
`/clusterState` endpoint (#3989)
add 4de7d56fc4 HDDS-7394. OM RPC FairCallQueue decay decision metrics list
caller username in the metric (#3878)
new 564f4d2faf HDDS-5366. [Ozone-Streaming] Implement stream method to
ContainerStateMachine. (#2358). Contributed by mingchao zhao
new bf191c78ce HDDS-5452. Add link method to ContainerStateMachine for
Ratis streaming (#2422)
new f06e763724 HDDS-5481. Fix stream() and link() method in
ContainerStateMachine. (#2451)
new 0c4105ae24 HDDS-5480. [Ozone-Streaming] Client and server should
support stream setup. (#2452)
new eef6601605 HDDS-5488. [Ozone-Streaming] Add a new
BlockOutputStream/KeyOutputStream to support streaming api (#2495)
new cd7cb217fd HDDS-5599. [Ozone-Streaming]drop BufferPool and
ChunkBuffer to avoid buffer copying (#2557)
new ae02e96e21 HDDS-5705. [Ozone-Streaming] Change ByteBufStreamOutput to
ByteBufferStreamOutput (#2603)
new 34e9fa357b HDDS-5742. Avoid unnecessary Bytebuffer conversions (#2673)
new c5ef6f7e70 HDDS-5486. [Ozone-Streaming] Streaming supports writing in
Pipline mode (#2682)
new bd0cf6bd06 HDDS-5849. [Ozone-Streaming]Write exceptions occur after
checksum is enabled (#2729)
new 1560b85072 HDDS-5674.[Ozone-Streaming] Handle client retries on
exception (#2701)
new 7e3607f841 HDDS-5895. [Ozone-Streaming] Make
raft.server.data-stream.client.pool.size configurable (#2766)
new f31738203e HDDS-5763. Provide an Executor for each LocalStream in
ContainerStateMachine (#2782)
new 7137fdcfa7 HDDS-5987. [Ozone-Streaming] Add XceiverClientRatis stream
config (#2841)
new ad2fcba931 HDDS-5961. [Ozone-Streaming] update the usage space of
Containers in the stream write (#2833)
new 3f1e2a5783 HDDS-5879. [Ozone-Streaming] OzoneBucket add the
createMultipartStreamKey method (#2760)
new 31f9ec6863 HDDS-5743. [Ozone-Streaming] Add option to write files via
streaming api in ofs and o3fs. (#2770)
new 2a56a60d7d HDDS-5851. [Ozone-Streaming] Define a PutBlock/maxBuffer
fixed boundary for streaming writes. (#2866)
new 67d79b5712 HDDS-6039. Define a minimum packet size during streaming
writes. (#2883)
new b12d8d487d HDDS-6130. [Ozone-Streaming] When releaseBuffers will get
“Couldn 't find the required future” (#2939)
new 750faff7ed HDDS-6139. [Ozone-Streaming] Fix incorrect computation of
totalAckDataLength. (#2978)
new 0d5d6e4155 HDDS-6178. [Ozone-Streaming] Fix NPE in HDDS-6139. (#2984)
new 4f7ecc42ac HDDS-6281. Update ratis version to 2.3.0-94db58b-SNAPSHOT
version (#3059)
new 3b1d2cfb63 HDDS-6138.[Ozone-Streaming] Define a limit on the size of
the retry bufferList. (#2946)
new e38639e469 HDDS-6298. Add XceiverServerRatis stream config (#3070)
new 818a7cc026 HDDS-5487. [Ozone-Streaming] BlockDataStreamOutput support
FlushDelay. (#3002)
new 933080773d HDDS-6282. Fix BlockDataStreamOutput#doFlushIfNeeded NPE
(#3060)
new fbb41ffc0c HDDS-6229. [Ozone-Streaming] Data Channel abstraction on
datanode (#3023)
new 12c016f70a HDDS-6355. [Ozone-Streaming] Fix CheckStyle problem (#3119)
new 06348e5756 HDDS-6388. [Ozone-Streaming] Streaming write support both
pipeline model and star model (#3145)
new 090c663e6b HDDS-5798. [Ozone-Streaming] Setup TlsConf parameters.
(#3207)
new 46c51847a6 HDDS-6137. [Ozone-Streaming] Refactor KeyDataStreamOutput.
(#3195)
new 20e96d83c3 HDDS-6500. [Ozone-Streaming] Buffer the PutBlockRequest at
the end of the stream. (#3229)
new 90a88a8120 HDDS-5666. Add option to createKey via streaming api in
Freon (#2574)
new f6333bd818 HDDS-6592. [Ozone-Streaming] Fix
ContainerStateMachine#applyTransaction assert error (#3315)
new 7e10a2cd4f HDDS-6842. [Ozone-Streaming] Reduce the number of watch
requests in StreamCommitWatcher. (#3492)
new 0b09d77372 HDDS-6867. [Ozone-Streaming] PutKeyHandler should not use
streaming to put EC key. (#3516)
new c5343bc73b HDDS-6955. [Ozone-streaming] Add explicit stream flag in
ozone shell (#3559)
new 4c63b2360a HDDS-7431. [Ozone-Streaming] Disable data steam by default.
(#3900)
new 3111030d17 HDDS-7438. [Ozone-Streaming] Add a createStreamKey method
to OzoneBucket. (#3914)
new 13ffca792c HDDS-7425. Add documentation for the new Streaming Pipeline
feature. (#3913)
new 226454dfa3 HDDS-7478. [Ozone-Streaming] NPE in when creating a file
with o3fs. (#3949)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c17be7013a)
\
N -- N -- N refs/heads/HDDS-4454 (226454dfa3)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 42 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/{post-commit.yml => ci.yml} | 11 +-
.github/workflows/post-commit.yml | 409 +--------------------
.../workflows/scheduled_ci.yml | 9 +-
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 56 ++-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 11 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 19 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 6 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 98 +++++
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 23 ++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 17 +-
.../hadoop/hdds/security/x509/SecurityConfig.java | 58 ++-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 +
.../apache/hadoop/ozone/common/StorageInfo.java | 4 +
.../ozone/container/common/helpers/ChunkInfo.java | 14 +
.../common/src/main/resources/ozone-default.xml | 65 +++-
.../hdds/datanode/metadata/CRLDBDefinition.java | 5 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 5 +
.../container/common/helpers/ContainerUtils.java | 29 ++
.../common/impl/ContainerLayoutVersion.java | 26 +-
.../container/common/impl/HddsDispatcher.java | 45 ++-
.../ozone/container/common/interfaces/Handler.java | 11 +
.../common/statemachine/DatanodeConfiguration.java | 6 +
.../common/statemachine/DatanodeStateMachine.java | 2 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 18 +-
.../reconstruction/ECContainerOperationClient.java | 40 +-
.../ECReconstructionCoordinator.java | 50 ++-
.../ozone/container/keyvalue/KeyValueHandler.java | 52 +++
.../background/BlockDeletingService.java | 16 +-
.../StaleRecoveringContainerScrubbingService.java | 5 +-
.../ozoneimpl/ContainerScannerConfiguration.java | 29 +-
.../ozoneimpl/OnDemandContainerScanner.java | 178 +++++++++
.../ozoneimpl/OnDemandScannerMetrics.java | 42 +++
.../ozone/container/ozoneimpl/OzoneContainer.java | 57 ++-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 2 +-
.../ozone/container/common/ContainerTestUtils.java | 39 +-
.../container/common/TestBlockDeletingService.java | 87 ++++-
...stStaleRecoveringContainerScrubbingService.java | 38 +-
.../container/common/impl/TestHddsDispatcher.java | 60 +++
.../impl/TestKeyValueStreamDataChannel.java | 2 +-
.../TestContainerScannerConfiguration.java | 14 +-
.../ozoneimpl/TestContainerScannerMetrics.java | 74 +++-
.../ozoneimpl/TestOnDemandContainerScanner.java | 157 ++++++++
.../dev-support/checkstyle/suppressions.xml | 1 +
.../io/erasurecode/rawcoder/package-info.java | 29 --
.../hadoop/hdds/security/ssl/KeyStoresFactory.java | 65 ++++
.../hdds/security/ssl/MonitoringTimerTask.java | 81 ++++
.../security/ssl/PemFileBasedKeyStoresFactory.java | 200 ++++++++++
.../hdds/security/ssl/ReloadingX509KeyManager.java | 179 +++++++++
.../security/ssl/ReloadingX509TrustManager.java | 153 ++++++++
.../hadoop/hdds/security/ssl}/package-info.java | 5 +-
.../certificate/authority/DefaultCAServer.java | 8 +-
.../x509/certificate/client/CertificateClient.java | 12 +-
.../client/CommonCertificateClient.java | 6 +
.../client/DefaultCertificateClient.java | 60 ++-
.../certificate/client/SCMCertificateClient.java | 3 +
.../certificates/utils/SelfSignedCertificate.java | 38 +-
.../hadoop/hdds/server/events/EventExecutor.java | 14 +
.../FixedThreadPoolWithAffinityExecutor.java | 52 +++
...entifiableEventPayload.java => IEventInfo.java} | 14 +-
.../hdds/server/http/PrometheusMetricsSink.java | 22 +-
.../hadoop/hdds/utils/DecayRpcSchedulerUtil.java | 121 ++++++
.../ssl/TestPemFileBasedKeyStoresFactory.java | 256 +++++++++++++
.../security/ssl/TestReloadingX509KeyManager.java | 83 +++++
.../ssl/TestReloadingX509TrustManager.java | 83 +++++
.../hdds/security/x509/CertificateClientTest.java | 73 ++--
.../certificate/authority/TestDefaultCAServer.java | 2 -
.../client/TestCertificateClientInit.java | 2 +-
.../client/TestDefaultCertificateClient.java | 52 +++
.../x509/certificate/utils/TestCRLCodec.java | 9 +-
.../certificate/utils/TestCertificateCodec.java | 27 +-
.../x509/certificates/TestRootCertificate.java | 24 +-
.../hdds/utils/TestDecayRpcSchedulerUtil.java | 75 ++++
.../src/main/proto/DatanodeClientProtocol.proto | 1 +
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 17 +-
.../scm/container/CloseContainerEventHandler.java | 5 +-
.../balancer/AbstractFindTargetGreedy.java | 23 +-
.../ContainerBalancerSelectionCriteria.java | 22 +-
.../container/balancer/ContainerBalancerTask.java | 107 ++++--
.../scm/container/balancer/FindSourceGreedy.java | 35 +-
.../scm/container/balancer/FindSourceStrategy.java | 11 +
.../FindTargetGreedyByNetworkTopology.java | 21 ++
.../balancer/FindTargetGreedyByUsageInfo.java | 23 ++
.../scm/container/balancer/FindTargetStrategy.java | 9 +
.../replication/ContainerCheckRequest.java | 6 +-
.../replication/ContainerHealthResult.java | 98 ++---
.../replication/ContainerReplicaPendingOps.java | 60 ++-
.../DatanodeCommandCountUpdatedHandler.java | 37 +-
.../replication/ECContainerReplicaCount.java | 23 ++
.../replication/ECOverReplicationHandler.java | 43 +--
.../replication/ECUnderReplicationHandler.java | 296 +++++++++------
.../replication/LegacyReplicationManager.java | 73 +---
.../replication/OverReplicatedProcessor.java | 34 +-
.../container/replication/ReplicationManager.java | 131 +++++--
.../replication/ReplicationManagerMetrics.java | 108 +++++-
.../container/replication/ReplicationQueue.java | 17 +
.../replication/UnderReplicatedProcessor.java | 46 +--
.../health/ClosedWithUnhealthyReplicasHandler.java | 127 +++++++
.../health/ECReplicationCheckHandler.java | 63 +++-
.../health/RatisReplicationCheckHandler.java | 54 +--
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 2 +-
.../hadoop/hdds/scm/node/DatanodeAdminMonitor.java | 2 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 65 +++-
.../hdds/scm/node/NodeDecommissionManager.java | 8 +-
.../hdds/scm/node/NodeDecommissionMetrics.java | 290 +++++++++++++++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 2 +
.../hdds/scm/safemode/SCMSafeModeManager.java | 4 -
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 32 +-
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 6 -
.../hdds/scm/server/StorageContainerManager.java | 135 ++-----
.../balancer/TestContainerBalancerTask.java | 5 +-
.../container/balancer/TestFindTargetStrategy.java | 34 ++
.../TestContainerReplicaPendingOps.java | 50 +++
.../TestDatanodeCommandCountUpdatedHandler.java} | 38 +-
.../replication/TestECContainerReplicaCount.java | 32 ++
.../replication/TestECOverReplicationHandler.java | 44 ++-
.../replication/TestECUnderReplicationHandler.java | 232 +++++++++++-
.../replication/TestLegacyReplicationManager.java | 37 +-
.../replication/TestOverReplicatedProcessor.java | 41 +--
.../replication/TestReplicationManager.java | 188 +++++++++-
.../replication/TestReplicationManagerMetrics.java | 2 +
.../replication/TestUnderReplicatedProcessor.java | 65 +---
.../TestClosedWithUnhealthyReplicasHandler.java | 194 ++++++++++
.../health/TestECReplicationCheckHandler.java | 262 ++++++++++++-
.../health/TestRatisReplicationCheckHandler.java | 70 ++--
.../scm/node/DatanodeAdminMonitorTestUtil.java | 141 +++++++
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 164 ++-------
.../hdds/scm/node/TestNodeDecommissionMetrics.java | 343 +++++++++++++++++
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 13 +
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 13 +-
hadoop-hdds/test-utils/pom.xml | 5 +
.../org/apache/ozone/test/GenericTestUtils.java | 41 +++
.../client/checksum/BaseFileChecksumHelper.java | 10 +-
.../client/checksum/ChecksumHelperFactory.java | 52 +++
.../client/checksum/ECBlockChecksumComputer.java | 221 +++++++++++
...ecksumHelper.java => ECFileChecksumHelper.java} | 171 ++++-----
.../ozone/client/io/ECBlockOutputStreamEntry.java | 34 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 234 ++++++++----
.../hadoop/ozone/client/TestOzoneECClient.java | 51 ++-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 11 -
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 65 +++-
.../hadoop/ozone/om/helpers/TestOmBucketArgs.java | 61 +++
.../dist/src/main/smoketest/ec/ozonefs.robot | 61 +++
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 26 ++
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 -
.../hadoop/fs/ozone/TestOzoneFileChecksum.java | 177 +++++++++
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 63 ++++
.../hadoop/ozone/TestStorageContainerManager.java | 51 +++
.../ozone/client/CertificateClientTestImpl.java | 17 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 2 -
.../client/rpc/TestBlockDataStreamOutput.java | 2 -
.../ozone/client/rpc/TestBlockOutputStream.java | 2 -
.../rpc/TestBlockOutputStreamFlushDelay.java | 2 -
.../rpc/TestBlockOutputStreamWithFailures.java | 2 -
...estBlockOutputStreamWithFailuresFlushDelay.java | 2 -
.../rpc/TestCloseContainerHandlingByClient.java | 2 -
.../client/rpc/TestContainerStateMachine.java | 2 -
.../TestContainerStateMachineFailureOnRead.java | 2 -
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../client/rpc/TestDeleteWithSlowFollower.java | 3 -
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 24 +-
.../rpc/TestMultiBlockWritesWithDnFailures.java | 2 -
...estOzoneClientRetriesOnExceptionFlushDelay.java | 4 -
.../rpc/TestOzoneClientRetriesOnExceptions.java | 3 -
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 -
.../ozone/container/TestECContainerRecovery.java | 128 ++++++-
.../commandhandler/TestBlockDeletion.java | 2 -
.../ozoneimpl/TestOzoneContainerWithTLS.java | 49 ++-
.../hadoop/ozone/dn/scanner/TestDataScanner.java | 181 ++++-----
.../ozone/om/TestOzoneManagerListVolumes.java | 48 ++-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 3 -
.../hadoop/ozone/om/TestSecureOzoneManager.java | 2 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 20 -
.../hadoop/ozone/shell/TestOzoneShellHA.java | 22 ++
.../java/org/apache/hadoop/ozone/om/OMStorage.java | 4 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 79 ++--
.../om/request/bucket/OMBucketCreateRequest.java | 4 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 34 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 75 +++-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 3 +-
.../key/OMDirectoriesPurgeResponseWithFSO.java | 15 +-
.../ozone/om/request/OMRequestTestUtils.java | 3 +
.../ozone/om/request/bucket/TestBucketRequest.java | 1 +
.../request/bucket/TestOMBucketCreateRequest.java | 46 +++
.../bucket/TestOMBucketCreateRequestWithFSO.java | 71 +++-
.../bucket/TestOMBucketSetPropertyRequest.java | 208 ++++++++++-
.../TestOMDirectoriesPurgeRequestAndResponse.java | 228 ++++++++++++
.../ozone/om/service/TestKeyDeletingService.java | 19 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 1 +
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 8 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 91 ++++-
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 44 ++-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 14 +-
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 32 --
.../org/apache/hadoop/ozone/recon/ReconServer.java | 36 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 6 +
.../ozone/recon/api/ClusterStateEndpoint.java | 23 +-
.../recon/api/types/ClusterStateResponse.java | 36 ++
.../hadoop/ozone/recon/scm/ReconStorageConfig.java | 4 +
.../scm/ReconStorageContainerManagerFacade.java | 25 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 3 +-
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 35 +-
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 33 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 21 +-
.../webapps/recon/ozone-recon-web/api/db.json | 241 ++++++------
.../webapps/recon/ozone-recon-web/api/routes.json | 6 +-
.../components/autoReloadPanel/autoReloadPanel.tsx | 18 +-
.../ozone-recon-web/src/utils/autoReloadHelper.tsx | 1 +
.../src/views/datanodes/datanodes.tsx | 10 +-
.../src/views/insights/insights.tsx | 10 +-
.../views/missingContainers/missingContainers.tsx | 115 +++---
.../src/views/overview/overview.tsx | 38 +-
.../src/views/pipelines/pipelines.tsx | 4 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 8 +-
.../recon/api/TestNSSummaryEndpointWithFSO.java | 7 +-
.../recon/api/TestTotalOpenContainerCount.java | 396 ++++++++++++++++++++
.../impl/TestOzoneManagerServiceProviderImpl.java | 70 +++-
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 10 +-
.../ozone/shell/bucket/ClearQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/bucket/SetQuotaHandler.java | 12 +-
.../ozone/shell/volume/ClearQuotaHandler.java | 12 +-
.../hadoop/ozone/shell/volume/SetQuotaHandler.java | 11 +-
pom.xml | 221 +----------
225 files changed, 9185 insertions(+), 2752 deletions(-)
copy .github/workflows/{post-commit.yml => ci.yml} (98%)
copy
hadoop-hdds/config/src/test/resources/META-INF/services/javax.annotation.processing.Processor
=> .github/workflows/scheduled_ci.yml (87%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerScanner.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandScannerMetrics.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
delete mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/package-info.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/KeyStoresFactory.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/MonitoringTimerTask.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/ReloadingX509KeyManager.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/ReloadingX509TrustManager.java
copy hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata
=>
framework/src/main/java/org/apache/hadoop/hdds/security/ssl}/package-info.java
(90%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/{IdentifiableEventPayload.java
=> IEventInfo.java} (85%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DecayRpcSchedulerUtil.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/TestPemFileBasedKeyStoresFactory.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/TestReloadingX509KeyManager.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/TestReloadingX509TrustManager.java
copy
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
=>
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java
(73%)
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestDecayRpcSchedulerUtil.java
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNewNodeHandler.java
=>
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/DatanodeCommandCountUpdatedHandler.java
(60%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosedWithUnhealthyReplicasHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionMetrics.java
copy
hadoop-hdds/server-scm/src/{main/java/org/apache/hadoop/hdds/scm/container/replication/InflightAction.java
=>
test/java/org/apache/hadoop/hdds/scm/container/replication/TestDatanodeCommandCountUpdatedHandler.java}
(52%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosedWithUnhealthyReplicasHandler.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/DatanodeAdminMonitorTestUtil.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeDecommissionMetrics.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ChecksumHelperFactory.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java
copy
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/{ReplicatedFileChecksumHelper.java
=> ECFileChecksumHelper.java} (65%)
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketArgs.java
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/ozonefs.robot
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileChecksum.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestTotalOpenContainerCount.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
