This is an automated email from the ASF dual-hosted git repository.
erose pushed a change to branch HDDS-10239-container-reconciliation
in repository https://gitbox.apache.org/repos/asf/ozone.git
from ab35173e24 HDDS-10376. Add a Datanode API to supply a merkle tree for
a given container. (#6945)
add e33fd2d7ca HDDS-10382. Optimize Netty memory allocation by avoiding
zero assignment in Java 9+ (#6975)
add 19f9afb42a HDDS-11215. Quota count can go wrong when double buffer
flush takes time. (#6978)
add c760804071 HDDS-11083. Avoid duplicate creation of
RunningDatanodeState (#6886)
add 324d29617e HDDS-11228. Ozone Recon HeatMap refactoring of code. (#6986)
add 96e1a8c446 HDDS-11140. Recon Disk Usage Metadata Details are not
working for du api (#6966)
add 86346cbf0c HDDS-10658. Include Transaction ID and Command Name in OM
Audit Messages. (#6949)
add 69ba680c51 HDDS-11136. Some containers affected by HDDS-8129 may still
be in the DELETING state incorrectly (#6967)
add 7a07625101 HDDS-11232. Spare InfoBucket RPC call for the
FileSystem#getFileStatus calls for the general case. (#6988)
add 9ba4a73f46 HDDS-11167. Use Key/TrustManagers directly for TLS
connection instead of factories (#6943)
add b3939ff0d4 HDDS-11241. Bump kotlin to 1.9.25 (#6996)
add b864195b09 HDDS-11242. Bump exec-maven-plugin to 3.3.0 (#6995)
add b07bb21333 HDDS-11223. Fix iteration over
ChunkBufferImplWithByteBufferList (#6999)
add 9bf587c829 HDDS-11245. Bump maven-core to 3.9.8 (#6997)
add 82c6bf3c82 HDDS-11023. Recon Disk Usage null conditions not handled
properly for null response (#6894)
add dcfa3b42ba HDDS-11238. Converge redundant getBucket calls for
FileSystem client delete (#6991)
add a532b8974b HDDS-11206. Statistics storage usage indicators include
min, max, median, stdev (#6977)
add f9d0bc1fa2 HDDS-11076. Revert HDDS-11076 and HDDS-11078 (#7004)
add dd831d8c9b HDDS-11221. Resolve potential time discrepancy for expired
multipart upload cleanup (#6982)
add 57bfa8cad1 HDDS-11236. Move Java version-specific NETTY_OPTS to
ozone-functions.sh (#7001)
add 99b481bcb7 HDDS-11082. Code cleanup in DatanodeStateMachine (#6883)
add b48a4c89e0 HDDS-11119. Unnecessary UPDATE_VOLUME audit entry for
DELETE_TENANT (#6994)
add 9533066d2c HDDS-11231. Make Recon start more resilient (#6987)
add 1ae270182d HDDS-11068. Move SstFiltered flag to a file in the snapshot
directory (#6965)
add a3f987ffe1 HDDS-10917. Refactor more tests from
TestContainerBalancerTask (#6734)
add cc95ee3efb HDDS-11078. Remove usage of sun.misc.Signal (#7006)
add 5118f23158 HDDS-11076. NoSuchMethodError: ByteBuffer.position
compiling with Java 9+, running with Java 8 (#6897)
add d38372ab97 HDDS-11201. Optimise FullTableCache eviction, scheduler and
lock. (#6962)
add 18437a3874 HDDS-11214. Added config to set rocksDB's max log file size
and num of log files (#7014)
add 311245b198 HDDS-11273. Bump commons-compress to 1.26.2 (#7023)
add 5a3b798662 HDDS-11235. Spare InfoBucket RPC call in FileSystem#mkdir()
call. (#6990)
add 15d803b9db HDDS-9915. [hsync] Interface to retrieve block info and
finalize block in DN through ratis. (#5783)
add b16087aa6c HDDS-9638. [hsync] File recovery support in OM (#5847)
add a281b932bb HDDS-10093. Make recoverLease call idempotent (#5958)
add 7224393a55 HDDS-9750. [hsync] Make Putblock performance acceptable -
Skeleton code (#5661)
add 1c20d84c93 HDDS-10104. [hsync]Introduce soft limit support for lease
recovery. (#5974)
add 082d759448 HDDS-8830. Add admin CLI to list open files (#5920)
add 04b6aa5e75 HDDS-10044. [hsync] File recovery support in Client (#5978)
add 305a176b7d HDDS-9751. [hsync] Make Putblock performance acceptable -
DataNode side (#5662)
add 54a75ca0a4 HDDS-9387. [hsync] Reduce updating block length times at OM
during hsync (#6054)
add 2e2d08e6dc HDDS-10141. [hsync] Support hard limit and auto recovery
for hsync file. (#6033)
add f5f737dc5c HDDS-10190. [hsync] Handle lease recovery for file without
blocks. (#6094)
add 2651d4c208 HDDS-9930. Remove open keys as well when keys are deleted
from KeyTable (#6079)
add b532f81e00 HDDS-10077. Add hsync metadata to hsync'ed keys in
OpenKeyTable as well (#6046)
add f065781956 HDDS-10147. Migrate TestFinalizeBlock to JUnit5. (#6133)
add 76a573a104 HDDS-9752. [hsync] Make Putblock performance acceptable -
Client side (#5663)
add c1d7b433d8 Merge remote-tracking branch 'asf/master' into HEAD
add 1f028f027b Checkstyle
add 8aa8a3658a Force testIgnoreExpiredRecoverhsyncKeys() to run first so
that the assertion is correction.
add 370b9d7c5c HDDS-10256. Retry block allocation when SCM is in safe
mode. (#6189)
add 0a1c5d4e4f HDDS-10242. [hsync] Handle penultimate block finalization.
(#6164)
add 463a09b7c6 HDDS-9884. Pass DatanodeVersion to the client (#6155)
add 9b134354c2 HDDS-10361. [hsync] Output stream should support direct
byte buffer. (#6256)
add d3c5cceb2f Merge remote-tracking branch 'asf/master' into HDDS-7593
add 1cf9e95eef HDDS-10427. Retry read wait based on policy. (#6292)
add f244a67037 HDDS-10252. [hsync] Revisit configuration keys for
incremental chunk list after HDDS-9884 (#6289)
add be5ffc29e2 HDDS-10471. [hsync] MockDatanodeStorage.writeChunk should
make a copy of byte string. (#6334)
add 1d81c32de9 Merge remote-tracking branch 'asf/HDDS-7593' into HDDS-7593
add 3cadb248f5 Merge remote-tracking branch 'asf/master' into HDDS-7593
add dd5b49bbeb HDDS-10497. [hsync] Refresh block token immediately if
block token expires. (#6371)
add 825c340bae HDDS-10511. Support ByteBufferPositionedReadable in
OzoneFSInputStream. (#6372)
add 5448ebceb6 Merge remote-tracking branch 'asf/master' into HDDS-7593
add 509c970054 HDDS-10442. [hsync] Add a Freon tool to measure client to
DataNode round-trip latency (#6297)
add 3fe5cdeac5 Merge remote-tracking branch 'asf/HDDS-7593' into
HDDS-7593-master-merge-97038ef
add 91e5d2eb28 HDDS-9130. [hsync] Combine WriteData and PutBlock requests
into one (#5980)
add 6a4e6993d3 HDDS-10564. Make Outputstream writeExecutor daemon threads.
(#6418)
add 6cfe9cf0a2 HDDS-10626. [LeaseRecovery] OM shuts down with 'SecretKey
client must have been initialized already' (#6467)
add d4314c94b6 HDDS-10591. [hsync] improve block token refresh message.
(#6444)
add f4d771647e Merge remote-tracking branch 'asf/master' into HEAD
add fd188d1ba1 HDDS-10650. Delete hsync key info from openFileTable while
deleting directory recursively. (#6495)
add 061c0a813e HDDS-10649. Handle auto hsync commit key having
directories. (#6530)
add 5b6be2b5b1 HDDS-10754. [hsync] lease recovery contract test class not
substantiated (#6638)
add d7e5b3a3fd HDDS-10770. [Hsync] Allow overwrite hsynced file (#6603)
add d993dffda6 HDDS-10774. Show deleted hsync keys in ListOpenFile CLI.
(#6628)
add 905834e0ee HDDS-10700. Merge HDDS-10780 into HBase feature branch.
(#6641)
add 3096887c5c HDDS-10838. Merge latest master change into HDDS-7593
feature branch.
add edcb2ac2a5 HDDS-10835. Show overwritten hsync keys in ListOpenFile CLI
(#6661)
add aa729597fb HDDS-10592. [hsync]Use Direct ByteBuffer in Output stream
lastChunkBuffer. (#6455)
add 1802c8753f HDDS-10860. Fix Intermittent failure in
TestLeaseRecovery.testFinalizeBlockFailure (#6707)
add 4a34792148 HDDS-10830. Replace ConcurrentHashMap with HashMap
protected by ReadWriteLock in NodeStateMap (#6654)
add bde7ee4d28 HDDS-10829. Suppress extra INFO log lines from ozone shell
commands output (#6657)
add 2cfdac6aa9 HDDS-10781. Do not use OFSPath in O3FS
BasicOzoneClientAdapterImpl (#6614)
add 28c9ea46a0 HDDS-10557. TestBlockOutputStream#testWriteExactlyFlushSize
is flaky (#6573)
add 29496dc62b HDDS-10742. Add option to close all pipelines (#6577)
add 1ee08b4241 HDDS-10801. Replace GSON with Jackson in hadoop-ozone
classes. (#6651)
add c1ab84e0d9 HDDS-10811. Reduce UTF8 string encoding by caching encoding
result (#6656)
add 309c3c4a9d HDDS-10847. Bump slf4j to 2.0.13 (#6670)
add 35715a48f6 HDDS-10848. Bump kotlin to 1.9.24 (#6671)
add 62a64c2531 HDDS-10849. Bump maven-source-plugin to 3.3.1 (#6672)
add 186f329b29 HDDS-10850. Bump commons-codec to 1.17.0 (#6669)
add 22ca84bbb2 HDDS-10851. Bump gradle-enterprise-maven-extension to
1.20.1 (#6668)
add 8c3dff264f HDDS-9039. Removed the pause and wait in RocksDB compaction
when tarball creation is in progress (#6552)
add ed5288ff5e HDDS-10273. Intermittent build failure while downloading
nodejs (#6664)
add e3cc4d9a06 HDDS-10702. Improve Recon startup failure handling and make
it more resilient. (#6583)
add f6710db9b7 HDDS-6717. Allow running flaky-test-check with specific
Ratis commit (#6675)
add 47942f7925 Revert "HDDS-10273. Intermittent build failure while
downloading nodejs (#6664)"
add 1db40bf516 HDDS-10845. Let some Freon generators use empty prefix by
default (#6673)
add 23cb3526fd HDDS-10371. NPE in OzoneAclUtils.isOwner (#6676)
add 3cf2a15578 HDDS-9679. Added Multipart Upload ID to S3MultiPart audit
logs. (#6677)
add b23674b2a6 HDDS-10559. Add a warning or a check to run repair tool as
System user (#6574)
add de228993e6 HDDS-9031. Upgrade acceptance tests to Docker Compose v2
(#6667)
add fb685f9709 HDDS-10859. Improve error messages when decommission and
maintenance fail-early (#6678)
add 6311f16b3b HDDS-10861. Ozone cli supports default ozone.om.service.id
(#6680)
add 86c6b39f6d HDDS-10699. Refactor ContainerBalancerTask and
TestContainerBalancerTask (#6537)
add c14f5113f2 HDDS-2643.
TestOzoneDelegationTokenSecretManager#testRenewTokenFailureRenewalTime fails
intermittently.
add ad0b688fe3 HDDS-6116. Remove flaky tag from TestSCMInstallSnapshot
(#6695)
add 5a49fc60d0 HDDS-10876. Bump jackson to 2.16.2 (#6697)
add 7a527c9d43 HDDS-10877. Bump Dropwizard metrics to 3.2.6 (#6699)
add 113944e822 HDDS-10878. Bump zstd-jni to 1.5.6-3 (#6701)
add 15a51fea04 HDDS-10514. Recon - Provide DN decommissioning detailed
status and info inline with current CLI command output. (#6376)
add e2dd311cab HDDS-10875. XceiverRatisServer#getRaftPeersInPipeline
should be called before XceiverRatisServer#removeGroup (#6696)
add bc3763fc43 HDDS-8752. Enable
TestOzoneRpcClientAbstract#testOverWriteKeyWithAndWithOutVersioning (#6702)
add cbc8797603 HDDS-10883. Improve logging in Recon for finalising DN
logic. (#6704)
add 64b7f01d55 HDDS-10435. Support S3 object tags for existing requests
(#6607)
add 3e20c4a36b HDDS-10832. Client should switch to streaming based on
OpenKeySession replication (#6683)
add 71509cb482 HDDS-10890. Increase default value for
hdds.container.ratis.log.appender.queue.num-elements (#6711)
add 6301bec130 HDDS-10899. Refactor Lease callbacks (#6715)
add f89601e5f3 HDDS-10422. Fix some warnings about exposing internal
representation in hdds-common (#6351)
add 040aebf655 HDDS-10897. Refactor OzoneQuota (#6714)
add e49f80509d HDDS-10925. [hsync] Parameterize TestBlockOutputStream on
ozone.client.stream.putblock.piggybacking (#6750)
add eff21da97e HDDS-9987. [hsync] Client side metrics. (#6468)
add 16c3f094d6 HDDS-10632. Handle inconsistent read issue for hsync keys
after lease recovery. (#6810)
add 88fa26b140 HDDS-9918. [hsync] Remove block token from Ratis log once
verified. (#6745) (#6814)
add dbd393912f Merge commit '787c9d3e135f6f17a18938f141b874c239eb2f76'
into HEAD
add 6914104282 HDDS-10934. Refactor TestOzoneRpcClient hierarchy (#6747)
add 7fa4d791a2 HDDS-10572. Implement multiDelete using OMKeysDeleteRequest
(#6751)
add fc6f409960 HDDS-10958. Bump maven-enforcer-plugin to 3.5.0 (#6759)
add 1274ea7cac HDDS-10959. Bump maven-shade-plugin to 3.6.0 (#6760)
add d4ac3854a2 HDDS-10960. Bump jersey2.version to 2.43 (#6762)
add 2ade7faa94 HDDS-10918. NPE in OM when leader transfers (#6735)
add ce5bbb0f4a HDDS-10976. Fix outdated reference to Ozone Datanode
running within HDFS (#6767)
add a37c979002 HDDS-10937. Ozone Recon - Handle startup failure and log
reasons as error due to SCM non-HA scenario (#6752)
add 65efe49905 HDDS-10972. Reduce the default watch timeout configuration
in DatanodeRatisServerConfig (#6772)
add 39baf0fdc6 HDDS-10869. SCMNodeManager#getUsageInfo memory occupancy
optimization (#6737)
add 19d4419d72 HDDS-10871. ContainerBalancerSelectionCriteria memory
occupancy optimization (#6738)
add 7745a26021 HDDS-10973. Fix order of assertion arguments in
TestContainerStateManagerIntegration (#6766)
add 6380532b06 HDDS-10967. Do Not Depend on Contiguous IDs for Proto Enum
Initialization. (#6775)
add 961c3f2f88 HDDS-10273. Intermittent build failure while downloading
NodeJS (#6686)
add ce09c4f3ac HDDS-10971. Replace ConcurrentHashMap with HashMap in
PipelineStateMap (#6765)
add bd8f5376bb HDDS-10991. Bump maven-checkstyle-plugin to 3.4.0 (#6788)
add 1f97d6e5e0 HDDS-10992. Bump assertj-core to 3.26.0 (#6790)
add a82a43b580 HDDS-10995. Bump compile-testing to 0.21.0 (#6791)
add ac25066fdd HDDS-10996. Bump commons-cli to 1.8.0 (#6792)
add 2f4af39ca6 HDDS-10986. Publish SBOM artifacts (#6785)
add cc126e0bb8 HDDS-10987. Add explicit dependency for hadoop-auth (#6787)
add 36aac4b8b1 HDDS-10988. Let zero OzoneQuota use byte as unit (#6786)
add 9e8faf91cc HDDS-10975. Recon Show SCM and OM services IDs in overview
page (#6769)
add 4f02853396 HDDS-10870. moveSelectionToFutureMap cleanup when future
complete (#6746)
add 2b67ce57af HDDS-10990. Fix memory leak in native lib (#6793)
add 25c9eb1e82 HDDS-9626. [Recon] Disk Usage page with high number of
key/bucket/volume (#6535)
add fdb75cd839 HDDS-10999. Remove dependency on ratis-server from Ozone
Client (#6800)
add d8a99c1d33 HDDS-10997. Avoid publishing build scan for helper Maven
commands (#6794)
add 0c47b0f3bf HDDS-10889. Remove certificate revocation related code.
(#6725)
add 463c9d3026 HDDS-10940. Remove useless
TestSCMContainerPlacementPolicyMetrics (#6749)
add fc6c35588c HDDS-10994. Migrate from Gradle Enterprise to Develocity
(#6795)
add bcf76588b0 HDDS-10640. Support x-amz-mp-parts-count for MPU key (#6798)
add a684901839 HDDS-10295. Provide an "ozone repair" subcommand to update
the snapshot info in transactionInfoTable (#6533)
add 4ba5079760 HDDS-11001. Speed up some tests that restart datanode
(#6803)
add 0499777509 HDDS-11002. Speed up TestPipelineClose (#6804)
add 693a9e5ea0 HDDS-11004. Keep logs only for failed iterations in
`flaky-test-check` (#6807)
add 3c358eab7b HDDS-10880. Duplicate Pipeline ID Detected in
ReconContainerManager. (#6742)
add 841e7a7b94 HDDS-11005. TestEndPoint#testRegisterRpcTimeout fails when
run in itself (#6806)
add 86027de410 HDDS-11015. Bump maven-install-plugin to 3.1.2 (#6818)
add 30978110a6 HDDS-10071. Merge TestContainerStateMachineIdempotency into
TestContainerOperations (#6813)
add a86732890d HDDS-11014. [hsync] Block finalization should also merge
last chunk to blockDataTable. (#6847)
add fd286447fa HDDS-10924. TestSCMHAManagerImpl#testAddSCM fails on ratis
master (#6753) (#6874)
add ee2bc3e46f HDDS-10910. Bump Ratis to 3.1.0 (#6872) (#6880)
add f7610c0012 Merge remote-tracking branch 'asf/master' into HDDS-7593
add 2b8f4128ae Merge remote-tracking branch 'asf/master' into HDDS-7593
add aa3c24e92a HDDS-10573. [hsync] Add a freon tool to benchmark
hsync/write concurrency (#6925)
add d8ea69dae7 HDDS-9844. [hsync] De-synchronize write APIs (#6859)
add 6c6dc4352e HDDS-11135. [hsync] Replace expensive
VolumeUsage.getMinVolumeFreeSpace() (#6930)
add e79dc66529 HDDS-11168. [hsync] Instantiates audit parameter lazily in
DataNode dispatch handler (#6933)
add f3b90fb6df HDDS-11185. Fix ContainerOpsLatencies metrics (#6947)
add 890c48dd7e HDDS-11193. KeyOutputStream flakiness when running write
and hsync concurrently (#6968)
add cc66490524 HDDS-11258. [hsync] Add new OM layout version. (#7013)
add 7828ba3ea2 HDDS-11260. [hsync] Add Ozone Manager protocol version
(#7015)
add a33af1dbce HDDS-11226. Make ExponentialBackoffPolicy maxRetries
configurable (#6985)
add 9e5da198a8 HDDS-11259. [hsync] DataNode should verify HBASE_SUPPORT
layout version for every PutBlock. (#7012)
add 97bc6beda0 HDDS-11224. Increase hdds.datanode.handler.count (#7011)
add c77115abd1 HDDS-11225. Increase ipc.server.read.threadpool.size (#7007)
add 4076ecb5a9 HDDS-10733. Merge HDDS-7593 (related to hsync and lease
recovery) feature branch into master.
add 1bb86206c4 HDDS-11280. Add Synchronize in
AbstractCommitWatcher.addAckDataLength (#7032)
add b5cbf9b04e HDDS-11274. Replace Hadoop annotations/configs with
Ozone-specific ones (#7026)
add 96bb60ede6 HDDS-11274. (addendum) Replace Hadoop annotations/configs
with Ozone-specific ones
add e1aa74cab7 HDDS-11278. Move code out of Hadoop util package (#7028)
add 2c0e0b1563 HDDS-11272. Statistics some node status information (#7025)
add 91b2b9139e HDDS-11270. [hsync] Add DN layout version
(HBASE_SUPPORT/version 8) upgrade test. (#7021)
add b891f7c295 HDDS-10517. Recon - Add a UI component for showing DN
decommissioning detailed status and info (#6724)
add e091e47160 HDDS-11249. Bump ozone-runner to 20240729-jdk17-1 (#7003)
add 6569278b93 HDDS-11257. Ozone write does not work when http proxy is
set for the JVM. (#7036)
add 7481790a84 HDDS-11283. Refactor KeyValueStreamDataChannel to avoid
spurious IDE build issues (#7040)
add c1b67d4a40 HDDS-11287. Code cleanup in XceiverClientSpi (#7043)
add 51ba4c88ab HDDS-11289. Bump docker-maven-plugin to 0.45.0 (#7024)
new 98369a8343 Merge branch 'master' into
HDDS-10239-container-reconciliation
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hadoop/hdds/scm/ContainerClientMetrics.java | 42 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 12 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 2 +-
.../hadoop/hdds/scm/XceiverClientMetrics.java | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 4 +-
.../hdds/scm/storage/AbstractCommitWatcher.java | 13 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 90 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 651 +++++++++++---
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 186 ++--
.../hadoop/hdds/scm/storage/CommitWatcher.java | 44 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 17 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 23 +-
.../client/io/BlockInputStreamFactoryImpl.java | 2 +-
.../hdds/scm/storage/DummyBlockInputStream.java | 10 +-
.../storage/DummyBlockInputStreamWithRetry.java | 7 +-
.../hdds/scm/storage/TestBlockInputStream.java | 21 +-
.../hadoop/hdds/scm/storage/TestBufferPool.java | 120 ++-
.../org/apache/hadoop/hdds/DatanodeVersion.java | 2 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 7 +
.../hadoop/hdds/protocol/DatanodeDetails.java | 8 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 6 +-
.../hadoop/hdds/ratis/conf/RatisClientConfig.java | 15 +
.../RequestTypeDependentRetryPolicyCreator.java | 2 +
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 6 +-
.../apache/hadoop/hdds/scm/net/NetConstants.java | 2 +-
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 2 +-
.../ContainerCommandResponseBuilders.java | 35 +
.../hadoop/hdds/scm/storage/BlockLocationInfo.java | 10 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 43 +-
.../security/ssl/PemFileBasedKeyStoresFactory.java | 165 ----
.../hdds/security/ssl/ReloadingX509KeyManager.java | 96 ++-
.../security/ssl/ReloadingX509TrustManager.java | 82 +-
.../security/token/ContainerTokenIdentifier.java | 2 +-
.../security/token/OzoneBlockTokenIdentifier.java | 2 +-
.../x509/certificate/client/CertificateClient.java | 13 +-
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 10 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 6 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 1 +
.../common/ChunkBufferImplWithByteBuffer.java | 4 +-
.../common/ChunkBufferImplWithByteBufferList.java | 41 +-
.../hadoop/{ => ozone}/util/CacheMetrics.java | 2 +-
.../hadoop/{ => ozone}/util/ClosableIterator.java | 2 +-
.../apache/hadoop/{ => ozone}/util/MetricUtil.java | 13 +-
.../hadoop/{ => ozone}/util/MutableMinMax.java | 6 +-
.../{ => ozone}/util/PerformanceMetrics.java | 2 +-
.../util/PerformanceMetricsInitializer.java | 2 +-
.../hadoop/{ => ozone}/util/ProtobufUtils.java | 2 +-
.../{ => ozone}/util/StringWithByteString.java | 2 +-
.../apache/hadoop/{ => ozone}/util/UUIDUtil.java | 2 +-
.../java/org/apache/hadoop/util/package-info.java | 23 -
.../common/src/main/resources/ozone-default.xml | 32 +-
.../hdds/ratis/conf/TestRatisClientConfig.java | 5 +
.../hadoop/hdds/utils/TestProtobufUtils.java | 7 +-
.../TestChunkBufferImplWithByteBufferList.java | 70 ++
.../ozone/container/ContainerTestHelper.java | 45 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 31 +-
.../org/apache/hadoop/ozone/audit/DNAction.java | 1 +
.../checksum/ContainerChecksumTreeManager.java | 2 +-
.../container/common/helpers/ContainerMetrics.java | 11 +-
.../container/common/helpers/DatanodeIdYaml.java | 2 +-
.../container/common/impl/HddsDispatcher.java | 60 +-
.../common/impl/StorageLocationReport.java | 2 +-
.../ozone/container/common/interfaces/Handler.java | 4 +
.../common/statemachine/DatanodeStateMachine.java | 29 +-
.../common/statemachine/StateContext.java | 18 +-
.../container/common/states/DatanodeState.java | 6 +
.../states/datanode/RunningDatanodeState.java | 90 +-
.../common/transport/server/XceiverServerGrpc.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 54 +-
.../transport/server/ratis/XceiverServerRatis.java | 11 +-
.../common/volume/AvailableSpaceFilter.java | 2 +-
.../ozone/container/common/volume/VolumeUsage.java | 72 +-
.../container/keyvalue/KeyValueContainer.java | 7 +
.../container/keyvalue/KeyValueContainerData.java | 34 +
.../ozone/container/keyvalue/KeyValueHandler.java | 132 ++-
.../keyvalue/helpers/KeyValueContainerUtil.java | 23 +
.../container/keyvalue/impl/BlockManagerImpl.java | 89 +-
.../ozone/container/keyvalue/impl/Buffers.java | 115 +++
.../keyvalue/impl/ChunkManagerDispatcher.java | 6 +
.../keyvalue/impl/FilePerBlockStrategy.java | 22 +
.../keyvalue/impl/KeyValueStreamDataChannel.java | 94 +-
.../keyvalue/impl/StreamDataChannelBase.java | 4 +-
.../keyvalue/interfaces/BlockManager.java | 3 +
.../keyvalue/interfaces/ChunkManager.java | 5 +
.../metadata/AbstractDatanodeDBDefinition.java | 7 +
.../container/metadata/AbstractDatanodeStore.java | 138 ++-
.../metadata/DatanodeSchemaOneDBDefinition.java | 6 +
.../metadata/DatanodeSchemaThreeDBDefinition.java | 36 +-
.../metadata/DatanodeSchemaTwoDBDefinition.java | 33 +-
.../ozone/container/metadata/DatanodeStore.java | 49 ++
.../metadata/DatanodeStoreSchemaThreeImpl.java | 9 +-
.../metadata/DatanodeStoreSchemaTwoImpl.java | 2 +-
.../DatanodeStoreWithIncrementalChunkList.java | 237 ++++++
.../container/ozoneimpl/ContainerController.java | 23 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 4 +-
.../replication/GrpcReplicationClient.java | 9 +-
.../container/replication/ReplicationServer.java | 6 +-
.../common/volume/TestReservedVolumeSpace.java | 26 +
.../container/keyvalue/TestKeyValueHandler.java | 8 +
.../TestKeyValueHandlerWithUnhealthyContainer.java | 14 +
.../keyvalue/impl/TestBlockManagerImpl.java | 186 +++-
.../impl/TestKeyValueStreamDataChannel.java | 1 -
.../upgrade/TestDatanodeUpgradeToHBaseSupport.java | 332 ++++++++
hadoop-hdds/docs/content/tools/Admin.md | 141 ++-
.../hdds/protocol/SecretKeyProtocolDatanode.java | 2 +-
.../protocolPB/SecretKeyProtocolDatanodePB.java | 2 +-
...inerLocationProtocolClientSideTranslatorPB.java | 2 +-
.../hdds/security/symmetric/ManagedSecretKey.java | 2 +-
.../client/DefaultCertificateClient.java | 49 +-
.../hdds/security/x509/keys/SecurityUtil.java | 35 -
.../apache/hadoop/hdds/utils/FaultInjector.java | 19 +
.../org/apache/hadoop/hdds/utils/SignalLogger.java | 17 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 4 +
.../hadoop/hdds/utils/db/RocksDBConfiguration.java | 30 +
.../hadoop/hdds/utils/db/cache/FullTableCache.java | 97 ++-
.../security/ssl/TestReloadingX509KeyManager.java | 14 +-
.../ssl/TestReloadingX509TrustManager.java | 9 +-
...ctory.java => TestSSLConnectionWithReload.java} | 54 +-
.../client/CertificateClientTestImpl.java | 61 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 34 +-
.../src/main/proto/DatanodeClientProtocol.proto | 31 +-
.../interface-client/src/main/proto/hdds.proto | 1 +
.../db/managed/ManagedRawSSTFileIterator.java | 2 +-
.../ozone/compaction/log/CompactionFileInfo.java | 2 +-
.../ozone/compaction/log/CompactionLogEntry.java | 2 +-
.../ozone/rocksdb/util/SstFileSetReader.java | 2 +-
.../container/AbstractContainerReportHandler.java | 12 +-
.../hdds/scm/container/ContainerManager.java | 10 +
.../hdds/scm/container/ContainerManagerImpl.java | 15 +
.../hdds/scm/container/ContainerStateManager.java | 12 +
.../scm/container/ContainerStateManagerImpl.java | 23 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 8 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 11 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 6 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 13 +
.../hdds/scm/node/NodeDecommissionManager.java | 6 +-
.../hadoop/hdds/scm/node/NodeManagerMXBean.java | 5 +
.../hadoop/hdds/scm/node/NodeStateManager.java | 52 ++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 92 ++
.../SecretKeyProtocolServerSideTranslatorPB.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 2 +-
.../main/resources/webapps/scm/scm-overview.html | 56 ++
.../src/main/resources/webapps/scm/scm.js | 39 +
.../scm/container/TestContainerManagerImpl.java | 58 ++
.../scm/container/TestContainerReportHandler.java | 106 +++
.../scm/container/TestContainerStateManager.java | 44 +
.../TestContainerBalancerDatanodeNodeLimit.java | 275 +++++-
.../balancer/TestContainerBalancerTask.java | 259 ------
.../scm/ha/TestInterSCMGrpcProtocolService.java | 86 +-
.../java/org/apache/hadoop/test/package-info.java | 22 -
.../{hadoop => ozone}/test/JacocoServer.java | 2 +-
.../java/org/apache/ozone/test/MetricsAsserts.java | 2 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 3 +-
.../client/io/BlockOutputStreamEntryPool.java | 35 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 10 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 52 +-
.../ozone/client/protocol/ClientProtocol.java | 24 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 32 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 131 ++-
.../hadoop/ozone/client/MockOmTransport.java | 50 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 37 +-
.../TestBlockOutputStreamIncrementalPutBlock.java | 160 ++++
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 8 +
.../hadoop/ozone/om/exceptions/OMException.java | 6 +-
.../{OpenKeySession.java => LeaseKeyInfo.java} | 27 +-
.../ozone/om/helpers/ListOpenFilesResult.java | 115 +++
.../hadoop/ozone/om/helpers/OpenKeySession.java | 3 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 32 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 1 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 71 +-
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/ozone/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/test-all.sh | 2 +-
hadoop-ozone/dist/src/shell/ozone/ozone | 18 +-
.../dist/src/shell/ozone/ozone-functions.sh | 8 +
hadoop-ozone/fault-injection-test/pom.xml | 8 +
.../fs/ozone/AbstractOzoneFileSystemTest.java | 1 +
.../ozone/AbstractOzoneFileSystemTestWithFSO.java | 3 +-
.../ozone/AbstractRootedOzoneFileSystemTest.java | 1 +
.../AbstractRootedOzoneFileSystemTestWithFSO.java | 4 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 80 ++
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 941 +++++++++++++++++++--
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 487 ++++++++++-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 123 +++
.../ozone/contract/AbstractOzoneContractTest.java | 44 +
.../hadoop/hdds/scm/storage/TestCommitWatcher.java | 13 -
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 30 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 18 +
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 42 +
.../client/rpc/TestBlockDataStreamOutput.java | 27 +
.../ozone/client/rpc/TestBlockOutputStream.java | 94 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 44 +-
.../ozone/client/rpc/TestDatanodeVersion.java | 143 ++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 5 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 150 +++-
.../container/TestContainerReportHandling.java | 156 ++++
.../TestContainerReportHandlingWithHA.java | 168 ++++
.../apache/hadoop/ozone/container/TestHelper.java | 21 +-
.../commandhandler/TestFinalizeBlock.java | 307 +++++++
.../hadoop/ozone/debug/TestLeaseRecoverer.java | 25 +-
.../hadoop/ozone/freon/TestDNRPCLoadGenerator.java | 2 +-
...CLoadGenerator.java => TestHsyncGenerator.java} | 94 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 48 +-
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 5 +-
.../ozone/om/TestOmContainerLocationCache.java | 36 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 9 +-
.../snapshot/TestSnapshotBackgroundServices.java | 3 +-
.../TestReconInsightsForDeletedDirectories.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 387 ++++++++-
.../ozone/shell/TestOzoneShellHAWithFSO.java | 2 +
.../org/apache/hadoop/utils/FaultInjectorImpl.java | 95 +++
.../src/main/proto/OmClientProtocol.proto | 35 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 52 +-
.../org/apache/hadoop/ozone/audit/OMAction.java | 5 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 4 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 3 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 12 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 17 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 120 ++-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 2 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 7 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 140 ++-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 2 +-
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 2 +-
.../hadoop/ozone/om/SstFilteringService.java | 78 +-
.../hadoop/ozone/om/helpers/OMAuditLogger.java | 203 +++++
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 4 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 2 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 5 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 25 +-
.../hadoop/ozone/om/request/RequestAuditor.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 2 +-
.../om/request/bucket/OMBucketDeleteRequest.java | 2 +-
.../om/request/bucket/OMBucketSetOwnerRequest.java | 2 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 2 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 2 +-
.../bucket/acl/OMBucketRemoveAclRequest.java | 2 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 2 +-
.../om/request/file/OMDirectoryCreateRequest.java | 2 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 2 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 2 +-
.../request/file/OMFileCreateRequestWithFSO.java | 4 +-
.../ozone/om/request/file/OMFileRequest.java | 10 +-
.../om/request/file/OMRecoverLeaseRequest.java | 167 ++--
.../om/request/key/OMAllocateBlockRequest.java | 14 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 19 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 33 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 166 +++-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 139 ++-
.../ozone/om/request/key/OMKeyCreateRequest.java | 2 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 4 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 13 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 11 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 2 +-
.../ozone/om/request/key/OMKeySetTimesRequest.java | 2 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 23 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 2 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 17 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 2 +-
.../request/key/acl/OMKeyAddAclRequestWithFSO.java | 2 +-
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 2 +-
.../key/acl/OMKeyRemoveAclRequestWithFSO.java | 2 +-
.../om/request/key/acl/OMKeySetAclRequest.java | 2 +-
.../request/key/acl/OMKeySetAclRequestWithFSO.java | 2 +-
.../key/acl/prefix/OMPrefixAddAclRequest.java | 2 +-
.../key/acl/prefix/OMPrefixRemoveAclRequest.java | 2 +-
.../key/acl/prefix/OMPrefixSetAclRequest.java | 2 +-
.../S3ExpiredMultipartUploadsAbortRequest.java | 2 +-
.../S3InitiateMultipartUploadRequest.java | 2 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 2 +-
.../multipart/S3MultipartUploadAbortRequest.java | 2 +-
.../S3MultipartUploadCommitPartRequest.java | 2 +-
.../S3MultipartUploadCompleteRequest.java | 6 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 4 +-
.../om/request/s3/security/OMSetSecretRequest.java | 2 +-
.../om/request/s3/security/S3GetSecretRequest.java | 2 +-
.../request/s3/security/S3RevokeSecretRequest.java | 2 +-
.../s3/tenant/OMTenantAssignAdminRequest.java | 2 +-
.../tenant/OMTenantAssignUserAccessIdRequest.java | 2 +-
.../request/s3/tenant/OMTenantCreateRequest.java | 6 +-
.../request/s3/tenant/OMTenantDeleteRequest.java | 9 +-
.../s3/tenant/OMTenantRevokeAdminRequest.java | 2 +-
.../tenant/OMTenantRevokeUserAccessIdRequest.java | 2 +-
.../security/OMCancelDelegationTokenRequest.java | 4 +-
.../security/OMGetDelegationTokenRequest.java | 4 +-
.../security/OMRenewDelegationTokenRequest.java | 4 +-
.../request/snapshot/OMSnapshotCreateRequest.java | 2 +-
.../request/snapshot/OMSnapshotDeleteRequest.java | 2 +-
.../request/snapshot/OMSnapshotRenameRequest.java | 2 +-
.../om/request/upgrade/OMCancelPrepareRequest.java | 2 +-
.../request/upgrade/OMFinalizeUpgradeRequest.java | 2 +-
.../ozone/om/request/upgrade/OMPrepareRequest.java | 2 +-
.../ozone/om/request/util/OmKeyHSyncUtil.java | 56 ++
.../om/request/volume/OMVolumeCreateRequest.java | 2 +-
.../om/request/volume/OMVolumeDeleteRequest.java | 2 +-
.../om/request/volume/OMVolumeSetOwnerRequest.java | 2 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 2 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 2 +-
.../volume/acl/OMVolumeRemoveAclRequest.java | 2 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 2 +-
.../om/response/file/OMRecoverLeaseResponse.java | 21 +-
.../key/OMDirectoriesPurgeResponseWithFSO.java | 11 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 29 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 19 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 28 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 18 +-
.../om/response/key/OMKeysDeleteResponse.java | 20 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 13 +-
.../response/snapshot/OMSnapshotPurgeResponse.java | 32 +-
.../ozone/om/service/OpenKeyCleanupService.java | 17 +-
.../ozone/om/service/SnapshotDeletingService.java | 7 +-
.../hadoop/ozone/om/snapshot/PersistentList.java | 2 +-
.../hadoop/ozone/om/snapshot/PersistentMap.java | 2 +-
.../hadoop/ozone/om/snapshot/PersistentSet.java | 2 +-
.../ozone/om/snapshot/RocksDbPersistentList.java | 2 +-
.../ozone/om/snapshot/RocksDbPersistentMap.java | 2 +-
.../ozone/om/snapshot/RocksDbPersistentSet.java | 2 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 2 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 3 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 12 +-
.../protocolPB/OzoneManagerRequestHandler.java | 56 +-
.../apache/hadoop/ozone/om/TestChunkStreams.java | 2 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 153 +++-
.../om/ratis/TestOzoneManagerStateMachine.java | 42 +
.../ozone/om/request/OMRequestTestUtils.java | 2 +-
.../om/request/file/TestOMRecoverLeaseRequest.java | 391 ++++++++-
.../om/request/key/TestOMKeyCommitRequest.java | 12 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 11 +
...estS3InitiateMultipartUploadRequestWithFSO.java | 3 +-
.../om/response/key/TestOMKeyCommitResponse.java | 8 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 4 +-
.../om/response/key/TestOMKeysDeleteResponse.java | 2 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 2 +-
.../om/service/TestOpenKeyCleanupService.java | 160 +++-
.../om/service/TestSnapshotDeletingService.java | 19 +-
.../ozone/om/snapshot/SnapshotTestUtils.java | 2 +-
.../om/snapshot/TestRocksDbPersistentList.java | 2 +-
.../om/snapshot/TestRocksDbPersistentMap.java | 2 +-
.../om/snapshot/TestRocksDbPersistentSet.java | 2 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 2 +-
.../ozone/om/snapshot/TestSstFilteringService.java | 14 +-
.../hadoop/fs/ByteBufferPositionedReadable.java | 92 ++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 104 ++-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 261 ++++--
.../fs/ozone/BasicRootedOzoneFileSystem.java | 44 +-
.../hadoop/fs/ozone/CapableOzoneFSInputStream.java | 1 +
.../fs/ozone/LeaseRecoveryClientDNHandler.java | 123 +++
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 9 +-
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 49 +-
.../org/apache/hadoop/fs/ozone/OzoneFsDelete.java | 4 +-
.../hadoop/fs/ozone/OzonePathCapabilities.java | 1 +
.../java/org/apache/hadoop/fs/ozone/Statistic.java | 6 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 37 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 37 +-
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 39 +-
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 44 +-
.../ozone/recon/api/AccessHeatMapEndpoint.java | 11 +
.../ozone/recon/api/types/HealthCheckResponse.java | 71 ++
.../ozone/recon/heatmap/HeatMapServiceImpl.java | 10 +
.../ozone/recon/heatmap/IHeatMapProvider.java | 11 +
.../spi/impl/OzoneManagerServiceProviderImpl.java | 43 +
.../webapps/recon/ozone-recon-web/api/db.json | 210 ++++-
.../webapps/recon/ozone-recon-web/api/routes.json | 13 +-
.../src/components/overviewCard/overviewCard.tsx | 6 +-
.../recon/ozone-recon-web/src/utils/common.tsx | 16 +-
.../src/views/datanodes/datanodes.tsx | 81 +-
.../src/views/datanodes/decommissionSummary.tsx | 121 +++
.../src/views/diskUsage/diskUsage.tsx | 286 ++++---
.../src/views/overview/overview.tsx | 32 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 2 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 13 +
.../ozone/admin/om/ListOpenFilesSubCommand.java | 248 ++++++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 1 +
.../hadoop/ozone/freon/DNRPCLoadGenerator.java | 14 +-
.../freon/FollowerAppendLogEntryGenerator.java | 3 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../apache/hadoop/ozone/freon/HsyncGenerator.java | 154 ++++
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 +-
.../containergenerator/GeneratorDatanode.java | 6 +-
.../org/apache/hadoop/test/OzoneTestDriver.java | 61 --
pom.xml | 22 +-
388 files changed, 12437 insertions(+), 2923 deletions(-)
delete mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/{ =>
ozone}/util/CacheMetrics.java (99%)
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/{ =>
ozone}/util/ClosableIterator.java (96%)
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/{ =>
ozone}/util/MetricUtil.java (90%)
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/{ =>
ozone}/util/MutableMinMax.java (95%)
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/{ =>
ozone}/util/PerformanceMetrics.java (98%)
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/{ =>
ozone}/util/PerformanceMetricsInitializer.java (98%)
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/{ =>
ozone}/util/ProtobufUtils.java (97%)
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/{ =>
ozone}/util/StringWithByteString.java (97%)
rename hadoop-hdds/common/src/main/java/org/apache/hadoop/{ =>
ozone}/util/UUIDUtil.java (97%)
delete mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/util/package-info.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/Buffers.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStoreWithIncrementalChunkList.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToHBaseSupport.java
rename
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/ssl/{TestPemFileBasedKeyStoresFactory.java
=> TestSSLConnectionWithReload.java} (79%)
delete mode 100644
hadoop-hdds/test-utils/src/main/java/org/apache/hadoop/test/package-info.java
rename hadoop-hdds/test-utils/src/main/java/org/apache/{hadoop =>
ozone}/test/JacocoServer.java (99%)
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestBlockOutputStreamIncrementalPutBlock.java
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/{OpenKeySession.java
=> LeaseKeyInfo.java} (55%)
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListOpenFilesResult.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDatanodeVersion.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandling.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReportHandlingWithHA.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestFinalizeBlock.java
copy
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/{TestDNRPCLoadGenerator.java
=> TestHsyncGenerator.java} (50%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/utils/FaultInjectorImpl.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/helpers/OMAuditLogger.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OmKeyHSyncUtil.java
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ByteBufferPositionedReadable.java
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/LeaseRecoveryClientDNHandler.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/HealthCheckResponse.java
create mode 100644
hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/datanodes/decommissionSummary.tsx
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/ListOpenFilesSubCommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java
delete mode 100644
hadoop-ozone/tools/src/test/java/org/apache/hadoop/test/OzoneTestDriver.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
