This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
from ee2bc3e46f HDDS-10910. Bump Ratis to 3.1.0 (#6872) (#6880)
add 385c4ec6ca HDDS-10934. Refactor TestOzoneRpcClient hierarchy (#6747)
add 321781b690 HDDS-10958. Bump maven-enforcer-plugin to 3.5.0 (#6759)
add c8f979adc1 HDDS-10959. Bump maven-shade-plugin to 3.6.0 (#6760)
add 7e480f8e61 HDDS-10960. Bump jersey2.version to 2.43 (#6762)
add 82342c1029 HDDS-10918. NPE in OM when leader transfers (#6735)
add d2ee09fb51 HDDS-10976. Fix outdated reference to Ozone Datanode
running within HDFS (#6767)
add f3a0dbd61a HDDS-10937. Ozone Recon - Handle startup failure and log
reasons as error due to SCM non-HA scenario (#6752)
add d11752fb17 HDDS-10972. Reduce the default watch timeout configuration
in DatanodeRatisServerConfig (#6772)
add 23e350b238 HDDS-10869. SCMNodeManager#getUsageInfo memory occupancy
optimization (#6737)
add 0561a11f08 HDDS-10871. ContainerBalancerSelectionCriteria memory
occupancy optimization (#6738)
add f78754965d HDDS-10973. Fix order of assertion arguments in
TestContainerStateManagerIntegration (#6766)
add 9e0f9677d4 HDDS-10967. Do Not Depend on Contiguous IDs for Proto Enum
Initialization. (#6775)
add 4b8767a934 HDDS-10572. Implement multiDelete using OMKeysDeleteRequest
(#6751)
add 0f848e4e24 HDDS-10273. Intermittent build failure while downloading
NodeJS (#6686)
add 6d177d92f2 HDDS-10971. Replace ConcurrentHashMap with HashMap in
PipelineStateMap (#6765)
add e84a047a72 HDDS-10991. Bump maven-checkstyle-plugin to 3.4.0 (#6788)
add 9baa27ca4e HDDS-10992. Bump assertj-core to 3.26.0 (#6790)
add fe745dd545 HDDS-10995. Bump compile-testing to 0.21.0 (#6791)
add 1c7ccabe9c HDDS-10996. Bump commons-cli to 1.8.0 (#6792)
add 5600cb8e34 HDDS-10986. Publish SBOM artifacts (#6785)
add 48f5a54db5 HDDS-10987. Add explicit dependency for hadoop-auth (#6787)
add aae64b151c HDDS-10988. Let zero OzoneQuota use byte as unit (#6786)
add 68ec901cde HDDS-10975. Recon Show SCM and OM services IDs in overview
page (#6769)
add 0697045a59 HDDS-10870. moveSelectionToFutureMap cleanup when future
complete (#6746)
add 11c9d60eb8 HDDS-10990. Fix memory leak in native lib (#6793)
add 925cc080ee HDDS-9626. [Recon] Disk Usage page with high number of
key/bucket/volume (#6535)
add 9047dec34d HDDS-10999. Remove dependency on ratis-server from Ozone
Client (#6800)
add c34360d2ec HDDS-10997. Avoid publishing build scan for helper Maven
commands (#6794)
add 65800dbc52 HDDS-10889. Remove certificate revocation related code.
(#6725)
add 0d8d1522a7 HDDS-10940. Remove useless
TestSCMContainerPlacementPolicyMetrics (#6749)
add e45d5f208c HDDS-10994. Migrate from Gradle Enterprise to Develocity
(#6795)
add 5c0a5f5363 HDDS-10640. Support x-amz-mp-parts-count for MPU key (#6798)
add f33f2fbb9e HDDS-10295. Provide an "ozone repair" subcommand to update
the snapshot info in transactionInfoTable (#6533)
add c5532a8061 HDDS-11001. Speed up some tests that restart datanode
(#6803)
add 15506217ac HDDS-11002. Speed up TestPipelineClose (#6804)
add bebd4c0afd HDDS-11004. Keep logs only for failed iterations in
`flaky-test-check` (#6807)
add 2aa561783d HDDS-10880. Duplicate Pipeline ID Detected in
ReconContainerManager. (#6742)
add 15f966e0ac HDDS-11005. TestEndPoint#testRegisterRpcTimeout fails when
run in itself (#6806)
add 738847fca8 HDDS-11015. Bump maven-install-plugin to 3.1.2 (#6818)
add b20ceebae1 HDDS-10071. Merge TestContainerStateMachineIdempotency into
TestContainerOperations (#6813)
add 81bc179d1c HDDS-11013. Ensure version is always set in
ContainerCommandRequestProto (#6812)
add 0ce031d985 HDDS-9644. Fixed incorrect validation of path with
namespace du commands. (#6755)
add d68bd23341 HDDS-10852. Recon Heatmap - Not allowing user to select
timerange if no data available (#6674)
add 6e826d527c HDDS-10407. Introduce metrics for deleteKey operation in
SCM service. (#6439)
add b37fc2fb16 HDDS-11003. Eliminate code duplication in
XceiverClientGrpc#sendCommandAsync (#6805)
add 1cc1ac6c84 HDDS-11019. Remove unused property release-year from root
POM (#6826)
add b87f977ef2 HDDS-10854. Run full Ozone CI with specific Ratis commit
(#6822)
add b783c34321 HDDS-11022. Set default test exclusion (#6830)
add 9a4894cd44 HDDS-11006. Selective checks: integration skipped when
build not required (#6808)
add 7164564293 HDDS-2887. Add config to tune replication level of watch
requests in Ozone client (#6768)
add 44cb3a59ee HDDS-11032. Decommissioned datanodes shows up again after
removing in Recon Datanodes page. (#6833)
add 2e19063c4e HDDS-10942. OM decommission config support for default
serviceID. (#6834)
add 0db0c28f73 HDDS-10935. Create hdds-crypto-api and hdds-crypto-default
modules (#6770)
add 913343d500 HDDS-11024. Add endKey filter option to ozone debug ldb
scan (#6828)
add a8dd86ff46 HDDS-10855. Handle Null ParentKeyInfo Error in Recon
Namespace Summary Task. (#6679)
add b3d191bb18 HDDS-10486. Recon datanode UI to incorporate explicit
removal of DEAD (#6411)
add 59560a156d HDDS-8942. Intermittent failure in
ITestOzoneContractCreate#testSyncable (#6821)
add 769d09e515 HDDS-10983. EC Key read corruption when the replica index
of container in DN mismatches (#6779)
add 19d623c3ee HDDS-10888. Restrict X509CertificateHolder usage to the
bare minimum required. (#6802)
add 9f1f7ed238 HDDS-11025. Update some ZH docs (#6829)
add 91c9bea7d9 HDDS-11049. Bump commons-text to 1.12.0 (#6848)
add b33b7f58be HDDS-11050. Bump native-maven-plugin to 1.0-M1 (#6852)
add 28533f3799 HDDS-11051. Bump build-helper-maven-plugin to 3.6.0 (#6849)
add 7eb3b29763 HDDS-11055. Bump develocity-maven-extension to 1.21.5
(#6851)
add fe754c56c6 HDDS-11044. Recon Disk Usage need to remove tool tip (#6840)
add e6cadc05c2 HDDS-10813. Improve Recon Debuggability for Snapshot
Fetching, Sync Monitoring, and Permission Validation. (#6640)
add c6ab3a7274 HDDS-11062. Minor code cleanup in XceiverServerRatis (#6857)
add 39ed524a58 HDDS-11020. Implement RoundRobinPipelineChoosePolicy (#6832)
add dcf2bc79ab HDDS-11066. Fix inaccurate descriptions for http.auth.type
in ozone-default.xml (#6866)
add 271974568d HDDS-11031. Merge BaseApprover abstract class into
DefaultApprover (#6844)
add 08b1e3f746 HDDS-11030. Do not throw OperatorCreationException from
CertificateApprover#sign (#6843)
add edc8c196da HDDS-11059. Reduce OM DEBUG message (#6861)
add dc005f4115 HDDS-10910. Bump Ratis to 3.1.0 (#6872)
add aeff4ac831 HDDS-11054. native check fails with Java11+ (#6853)
add 50c5a78c79 HDDS-11080. Remove upgrade docs section: Features Requiring
Finalization (#6878)
add 7df3120ff0 HDDS-10864. Recon Disk Usage If one volume is large it
occupies almost all space of Pie Chart (#6801)
add 5bc56307a0 HDDS-11086. Bump maven-jar-plugin to 3.4.2 (#6890)
add b8e4e1d253 HDDS-11088. Bump junit to 5.10.3 (#6887)
add 261c9bd2b7 HDDS-11089. Bump license-maven-plugin to 2.4.0 (#6888)
add 950a4b55f8 HDDS-11084. Read SstFilteringService config only once in
SnapshotDeletingService (#6885)
add 173e7d26f5 HDDS-10508. OmUtils.getAllOMHAAddresses may throw NPE
(#6884)
add 9513e0bac6 HDDS-11042. CI with Ratis ignores Ozone ref (#6838)
add 70a73a193b HDDS-10932. Reduce number of watch requests by using
CommitInfoProto from NotReplicatedException (#6879)
add 148cbe4368 HDDS-11090. Add explicit dependency for jcip-annotations
(#6892)
add a8f13fde38 HDDS-11092. Disable ozonescripts test (#6893)
add 080698d360 HDDS-11094. Bump nimbus-jose-jwt to 9.40 (#6891)
add a09424ad9f HDDS-11067. Log Error in BackgroundService#PeriodicalTask
(#6867)
add 34362b07b3 HDDS-11095. Generate fixed length string with Robot builtin
(#6895)
add 3f14219639 HDDS-11078. Remove usage of sun.misc.Signal (#6876)
add e99e5d5dcc HDDS-11102. Bump jetty to 9.4.55.v20240627 (#6901)
add f2bb59b18c HDDS-11104. Bump maven-dependency-plugin to 3.7.1 (#6903)
add 869ca8f5b2 HDDS-11101. Use OZONE_RUNNER_IMAGE for httpfs (#6906)
add 79cdfc95cf HDDS-11114. Bump maven-javadoc-plugin to 3.7.0 (#6905)
add 8f610da91f HDDS-11076. NoSuchMethodError: ByteBuffer.position
compiling with Java 9+, running with Java 8 (#6897)
add 56d1289b17 HDDS-9874. Introduce Metrics for listKeys Dashboard (#5745)
add d6d33f6c51 HDDS-9977. Dashboard for create key metrics (#6865)
add 3e70cf4165 HDDS-11106. Save logs for stopped containers (#6908)
add f7610c0012 Merge remote-tracking branch 'asf/master' into HDDS-7593
No new revisions were added by this update.
Summary of changes:
.github/workflows/build-ratis.yml | 2 +-
.github/workflows/ci-with-ratis.yml | 53 +
.github/workflows/ci.yml | 103 +-
.mvn/extensions.xml | 2 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 68 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 106 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 17 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 25 +-
.../ozone/client/io/BlockInputStreamFactory.java | 3 +-
.../client/io/BlockInputStreamFactoryImpl.java | 3 +-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 7 +-
.../hdds/scm/storage/DummyBlockInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 2 +-
.../hdds/scm/storage/TestBlockInputStream.java | 2 +-
.../storage/TestBlockOutputStreamCorrectness.java | 10 +-
.../client/io/TestBlockInputStreamFactoryImpl.java | 11 +-
hadoop-hdds/common/pom.xml | 4 +
.../java/org/apache/hadoop/hdds/StringUtils.java | 79 -
.../org/apache/hadoop/hdds/client/BlockID.java | 57 +-
.../hdds/ratis/ContainerCommandRequestMessage.java | 4 +
.../hadoop/hdds/ratis/conf/RatisClientConfig.java | 18 +
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 3 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 60 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 15 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 47 +-
.../hadoop/hdds/security/SecurityConfig.java | 36 +-
.../x509/certificate/utils/CertificateCodec.java | 107 +-
.../certificate/utils/SelfSignedCertificate.java | 24 +-
.../hadoop/hdds/utils/BackgroundService.java | 7 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 4 +
.../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 5 +-
.../common/src/main/resources/ozone-default.xml | 12 +-
.../ratis/TestContainerCommandRequestMessage.java | 3 +
.../hdds/security/x509/CertificateTestUtils.java | 1 +
.../ozone/container/ContainerTestHelper.java | 22 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 3 +-
.../transport/server/ratis/XceiverServerRatis.java | 26 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 23 +-
.../container/keyvalue/helpers/BlockUtils.java | 23 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 20 +-
.../keyvalue/interfaces/BlockManager.java | 4 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 59 +-
.../container/common/helpers/TestBlockData.java | 2 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 59 +
.../impl/TestKeyValueStreamDataChannel.java | 4 +-
hadoop-hdds/{annotations => crypto-api}/pom.xml | 34 +-
.../{annotations => crypto-default}/pom.xml | 34 +-
hadoop-hdds/docs/content/concept/Recon.zh.md | 2 +-
.../docs/content/feature/Nonrolling-Upgrade.md | 14 -
hadoop-hdds/docs/content/start/OnPrem.zh.md | 6 +-
hadoop-hdds/framework/pom.xml | 4 +
.../x509/certificate/authority/BaseApprover.java | 258 --
.../certificate/authority/CertificateApprover.java | 27 +-
.../certificate/authority/CertificateServer.java | 6 +-
.../certificate/authority/DefaultApprover.java | 217 +-
.../certificate/authority/DefaultCAServer.java | 107 +-
.../client/DefaultCertificateClient.java | 2 +-
.../certificate/client/SCMCertificateClient.java | 19 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 75 +
.../org/apache/hadoop/hdds/utils/SignalLogger.java | 30 +-
.../apache/hadoop/hdds/utils/db/RocksDatabase.java | 2 +-
.../x509/certificate/authority/MockApprover.java | 58 -
.../certificate/authority/TestDefaultCAServer.java | 80 +-
.../certificate/authority/TestDefaultProfile.java | 33 +-
.../client/CertificateClientTestImpl.java | 64 +-
.../client/TestDefaultCertificateClient.java | 12 +-
.../client/TestDnCertificateClientInit.java | 7 +-
.../client/TestRootCaRotationPoller.java | 15 +-
.../certificate/utils/TestCertificateCodec.java | 174 +-
.../certificate/utils/TestRootCertificate.java | 69 +-
hadoop-hdds/pom.xml | 2 +
hadoop-hdds/rocks-native/pom.xml | 5 -
.../placement/metrics/SCMPerformanceMetrics.java | 94 +
...cy.java => RoundRobinPipelineChoosePolicy.java} | 27 +-
.../hdds/scm/security/RootCARotationManager.java | 35 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 8 +
.../hdds/scm/server/StorageContainerManager.java | 19 +
.../scm/server/StorageContainerManagerStarter.java | 4 +-
.../hdds/scm/ha/io/TestX509CertificateCodec.java | 8 +
.../scm/metadata/TestX509CertificateCodec.java | 8 +
.../TestRoundRobinPipelineChoosePolicy.java | 170 +
.../scm/security/TestRootCARotationManager.java | 7 +-
hadoop-ozone/client/pom.xml | 5 +
.../client/checksum/ECFileChecksumHelper.java | 7 +-
.../checksum/ReplicatedFileChecksumHelper.java | 8 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 4 +-
.../hadoop/ozone/client/io/KeyInputStream.java | 8 +-
hadoop-ozone/common/pom.xml | 5 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 43 +-
hadoop-ozone/csi/pom.xml | 4 +
.../org/apache/hadoop/ozone/csi/CsiServer.java | 4 +-
hadoop-ozone/dev-support/checks/junit.sh | 2 +-
hadoop-ozone/dev-support/checks/native.sh | 2 +-
hadoop-ozone/dev-support/checks/unit.sh | 1 -
...Metrics.json => Ozone - CreateKey Metrics.json} | 4104 +++++++-------------
.../ozonescripts/{test.sh => disabled-test.sh} | 0
.../compose/ozonesecure-ha/docker-compose.yaml | 2 +-
.../main/compose/ozonesecure/docker-compose.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 6 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 11 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 12 +
.../src/main/smoketest/s3/MultipartUpload.robot | 2 +-
.../dist/src/main/smoketest/s3/objectcopy.robot | 4 +-
.../dist/src/main/smoketest/s3/objectputget.robot | 6 +-
.../hadoop/hdds/scm/TestXceiverClientGrpc.java | 22 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 9 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 155 +-
.../ozone/client/rpc/TestBlockOutputStream.java | 2 +-
...ayCommitInRatis.java => TestCommitInRatis.java} | 30 +-
.../client/rpc/TestFailureHandlingByClient.java | 19 +-
.../rpc/TestOzoneClientRetriesOnExceptions.java | 4 +-
.../ozone/client/rpc/TestWatchForCommit.java | 33 +-
.../ozone/container/TestContainerReplication.java | 195 +
.../org/apache/hadoop/ozone/debug/TestLDBCli.java | 24 +
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 7 +-
.../hadoop/ozone/om/TestSecureOzoneManager.java | 4 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 20 +-
.../hadoop/ozone/om/OMPerformanceMetrics.java | 22 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 32 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 4 +-
.../ozone/om/service/SnapshotDeletingService.java | 14 +-
.../apache/hadoop/ozone/om/TestChunkStreams.java | 5 +-
.../om/service/TestSnapshotDeletingService.java | 100 +
.../ozone/om/snapshot/TestOmSnapshotUtils.java | 44 -
.../security/TestOmCertificateClientInit.java | 4 +-
.../TestOzoneDelegationTokenSecretManager.java | 8 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 3 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 26 +
.../hadoop/ozone/recon/api/NodeEndpoint.java | 6 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 125 +-
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 12 +-
.../ozone/recon/tasks/NSSummaryTaskWithOBS.java | 2 +
.../webapps/recon/ozone-recon-web/api/db.json | 242 +-
.../webapps/recon/ozone-recon-web/api/routes.json | 3 +-
.../src/utils/axiosRequestHelper.tsx | 15 +
.../src/views/datanodes/datanodes.less | 4 +
.../src/views/datanodes/datanodes.tsx | 84 +-
.../src/views/diskUsage/diskUsage.tsx | 42 +-
.../ozone-recon-web/src/views/heatMap/heatmap.tsx | 65 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 9 +-
.../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 3 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 101 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 3 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 4 -
.../ozone/admin/nssummary/NSSummaryAdmin.java | 6 +
.../ozone/admin/om/DecommissionOMSubcommand.java | 4 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 26 +-
pom.xml | 80 +-
148 files changed, 4144 insertions(+), 4603 deletions(-)
create mode 100644 .github/workflows/ci-with-ratis.yml
copy hadoop-hdds/{annotations => crypto-api}/pom.xml (54%)
copy hadoop-hdds/{annotations => crypto-default}/pom.xml (53%)
delete mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/BaseApprover.java
rename hadoop-hdds/{common =>
framework}/src/main/java/org/apache/hadoop/hdds/utils/SignalLogger.java (74%)
delete mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockApprover.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMPerformanceMetrics.java
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/choose/algorithms/{RandomPipelineChoosePolicy.java
=> RoundRobinPipelineChoosePolicy.java} (65%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/choose/algorithms/TestRoundRobinPipelineChoosePolicy.java
copy hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/{Ozone - JVM
Metrics.json => Ozone - CreateKey Metrics.json} (52%)
rename hadoop-ozone/dist/src/main/compose/ozonescripts/{test.sh =>
disabled-test.sh} (100%)
rename
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/{Test2WayCommitInRatis.java
=> TestCommitInRatis.java} (85%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestSnapshotDeletingService.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
