This is an automated email from the ASF dual-hosted git repository.
errose28 pushed a change to branch HDDS-14496-zdu
in repository https://gitbox.apache.org/repos/asf/ozone.git
from c064bafb5e1 Merge branch 'master' into HDDS-14496-zdu
add 36481b61e44 HDDS-16595. Bump download-maven-plugin to 2.1.0 (#11316)
add 119fc06fd8f HDDS-16613. Bump actions/labeler to 7.0.0 (#11336)
add f39a349e90d HDDS-16623. Datanode can be missing from NetworkTopology
after it recovers from DEAD (#11343)
add 4b020c16d65 HDDS-16597. Bump iceberg to 1.11.0 (#11318)
add 03d838e01e0 HDDS-16398. Fix intermittent AlreadyClosedException in
TestCommitWatcher (#11227)
add 8bdfef0b8fb HDDS-13365. OM support automatic table cache tracking and
cleanup (#10171)
add 7b7e453e9ab HDDS-16644. Bump jackson to 2.21.7 (#11352)
add ba05c2e3ad8 HDDS-16497. Support OM ThreadLocal context propagation for
Ratis read (#11257)
add 2468fa8b14d HDDS-16647. Set only server-side target version for
compile check (#11362)
add 907411afe71 HDDS-12950. Refactor TestBucketList to reduce duplicated
declarations (#11338)
add 3448f66a4df HDDS-16645. Replace usage of deprecated AccessController
in TestOmMixedWorkloadUnderDeletionBench (#11363)
add ce4953e56ed HDDS-16646. Remove listMaxSize from KeyLifecycleService.
(#11361)
add 12635659b13 HDDS-16605. Reduce noise from failed copy-jar in
integration check (#11326)
add 46a0ec802b9 HDDS-16551. Remove duplicate heatmap feature check in
Recon v2 UI (#11323)
add b15c871db90 HDDS-16642. Hide heatmap navbar tab until feature status
is confirmed in Recon v2 (#11360)
add db63057eaf2 HDDS-16635. Remove redundant InfoBucket RPC from S3
GetBucketLifecycleConfiguration (#11355)
add 8638daf7f3e HDDS-16636. Remove redundant InfoBucket RPC from S3
ListParts (#11358)
add 24e06a40c16 HDDS-15835. Count incomplete MPU parts in quota repair
(#10795)
add 2b5874b743b HDDS-16226. Tool to obtain snapshot trapped bytes for a
bucket. (#11122)
add 4317f21bc85 HDDS-16603. Remove redundant infoBucket RPC call in S3
tagging APIs (#11339)
add a0119aaebd2 HDDS-16637. Remove redundant InfoBucket RPC from S3
ListMultipartUploads (#11359)
add 0f20ca7b287 HDDS-16655. Skip shading in integration (filesystem) check
(#11371)
add 8c249d0a048 HDDS-15142. Support HMAC-SHA256 trailer signature
verification for S3 chunked uploads (#11222)
add 985d7a94e54 HDDS-16099. Speed up TestStreamBlockInputStream (#11337)
add ecafca6f6f6 HDDS-16630. Update the network topology when a datanode
re-registers with new ports or a new version (#11353)
add d5d4df4d4b6 HDDS-16395. Add lock-free StreamBlock pread (#11314)
add 4a832912fea HDDS-16591. Reject not implemented S3 subresource APIs
(#11313)
add f35c48c9f64 HDDS-8280. Upgrade Jetty 9.4 to 12.0.38 (EE8) (#11367)
add aade154f8dd HDDS-16648. Tune HTTP settings for CI (#11375)
add 29758e4ac1d HDDS-16441. Orphaned snapshot diff task marks a
resubmitted job DONE with another run's metadata (#11321)
add 037a367ceb1 HDDS-16650. Reuse bucket metadata in S3 lifecycle PUT and
DELETE (#11372)
add fd90ad1b7a5 HDDS-16530. Allow Recon service principal to call OM
getDBUpdates (#11382).
add f7bfdf19b63 HDDS-16659. Reuse destination OzoneBucket in streaming S3
CopyObject (#11383)
add 63f93f7ca79 HDDS-16323. StreamBlockInputStream: refill the pre-read
window in bulk instead of once per response (#11159)
add 60ce9947960 HDDS-16619. Intermittent failure in
TestReconAsPassiveScm.testDatanodeRegistrationAndReports (#11341)
add 177fe51472f HDDS-13303. Fix TestSnapshotDeletingServiceIntegrationTest
(#11310)
add 29b98532703 S3 header auth expiry in 15min and correct date validation.
add 03f73b4b07a HDDS-16705. Bump zstd-jni to 1.5.7-20 (#11390)
add ae281be3149 HDDS-16707. Bump commons-lang3 to 3.21.0 (#11394)
add 317c423af3e HDDS-16706. Bump slf4j to 2.0.20 (#11392)
add 225727d1890 HDDS-2821. Include datanode hostname/IP in logs in
addition to UUID (#11381)
add 898bb4310f2 HDDS-14337. Fix SCM startup failure near year boundary
when seeding localId (#11184)
add 7f7cb3f5597 HDDS-14018. Avoid repeated operations in
SCMNodeManager#getNodeStatistics (#11292)
add 69c51c4c2ca HDDS-16710. Bump axios to 1.20.0 (#11378)
add 2e51e156477 HDDS-16711. Bump moment to 2.31.0 (#11373)
add ac9a57ddc1b HDDS-16658. Reuse source key details when opening input
stream in S3 CopyObject (#11396)
add 4c99fd8fe12 HDDS-16654. Replace usage of deprecated finalize() in OM
(#11376)
add 77d582975a6 HDDS-16631. Fix chooseRandom for rack names with common
prefixes (#11401)
add 037f9161a33 HDDS-16704. ReadBlock fails with IllegalStateException
when a response is shorter than responseDataSize (#11402)
add 65c4078829d HDDS-15089. Support S3 per request read consistency
(#11252)
add ec64e924c40 HDDS-16300. Allow OM to dynamically reconfigure its SCM
node list without a restart (#11218)
add 4d1e5d7b49e HDDS-16713. Bump develocity-maven-extension to 2.6.0
(#11405)
add 6772ee33dc3 HDDS-16708. Let dependabot ignore iceberg minor version
upgrades (#11398)
add 3afe9a108f5 HDDS-16673. Avoid redundant ListBuckets RPCs when S3
bucket listing reaches the end (#11397)
add b79d76542ed HDDS-16674. Bump awssdk to 2.55.6 (#11407)
add 4066b0e8e23 HDDS-15991. Speed up deleted table scans in quota repair
(#11386)
add 1ad6848c3c4 HDDS-16241. gRPC deadline kills long-lived block streams
after 30 seconds and the client never recovers (#11080)
add 7ed017d22e4 HDDS-16721. StreamBlockInputStream.read() returns a
negative value for bytes 0x80 to 0xFF (#11411)
add 646c4dfd8c1 HDDS-16643. Remove CleanupTableInfo mechanism (#11365)
add 96c43b3354b HDDS-16362. GetObjectAttributes ObjectParts should return
Part entries for FSO buckets (#11242).
add ccce9187dcb HDDS-15749. Run specific JUnit tests if possible (#10671)
add fb0adda14fc HDDS-16666. Flush SCM transaction in memory during apply
transaction (#11409)
add 7d9e8417e83 HDDS-16008. PutBlocks from Flushes also go without Raft
(#11356)
add a99fabb9dea HDDS-16716. Add description for
ozone.scm.ec.pipeline.per.volume.factor (#11415)
new a9a64dbed31 Merge branch 'master' into HDDS-14496-zdu
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/dependabot.yml | 8 +-
.github/workflows/build-ratis.yml | 2 +-
.github/workflows/check.yml | 2 +-
.github/workflows/ci.yml | 35 +-
.github/workflows/intermittent-test-check.yml | 2 +-
.github/workflows/label-pr.yml | 2 +-
.github/workflows/populate-cache.yml | 2 +-
.github/workflows/repeat-acceptance.yml | 2 +-
.mvn/extensions.xml | 2 +-
dev-support/ci/selective_ci_checks.bats | 42 +-
dev-support/ci/selective_ci_checks.sh | 68 +++-
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 23 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 5 +-
.../hdds/scm/storage/BlockDataStreamOutput.java | 77 +++-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hdds/scm/storage/MultipartInputStream.java | 48 +--
.../hadoop/hdds/scm/storage/PartInputStream.java | 13 +
.../hdds/scm/storage/StreamBlockInputStream.java | 182 +++++++--
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 10 +-
.../hdds/scm/storage/MockDatanodePipeline.java | 45 +-
.../scm/storage/TestBlockDataStreamOutput.java | 59 ++-
.../hdds/scm/storage/TestMultipartInputStream.java | 69 ++++
.../scm/storage/TestStreamBlockInputStream.java | 361 ++++++++++++++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +
.../hadoop/hdds/utils/ConnectionFailureUtils.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 23 +-
.../hdds/utils/TestConnectionFailureUtils.java | 14 +
.../common/statemachine/DatanodeConfiguration.java | 22 +
.../ClosePipelineCommandHandler.java | 4 +-
.../transport/server/GrpcXceiverService.java | 68 +++-
.../common/transport/server/XceiverServerGrpc.java | 5 +-
.../server/ratis/ContainerStateMachine.java | 31 +-
.../transport/server/ratis/DispatcherContext.java | 4 +-
.../common/transport/server/ratis/LocalStream.java | 22 +-
.../container/keyvalue/ReadBlockComputation.java | 8 +-
.../container/common/impl/TestHddsDispatcher.java | 3 +-
.../transport/server/TestGrpcXceiverService.java | 169 ++++++++
.../container/keyvalue/TestKeyValueHandler.java | 18 +
.../docs/content/feature/Reconfigurability.md | 2 +
hadoop-hdds/framework/pom.xml | 22 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 2 +-
.../scm/proxy/SCMFailoverProxyProviderBase.java | 91 ++++-
.../hdds/server/http/AdminAuthorizedServlet.java | 2 +-
.../hadoop/hdds/server/http/BaseHttpServer.java | 56 ++-
...horizedServlet.java => HddsJMXJsonServlet.java} | 28 +-
.../hadoop/hdds/server/http/HttpServer2.java | 244 ++++++++---
.../hdds/server/http/ServletElementsFactory.java | 4 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 27 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +
.../utils/db/cache/TableCacheUpdateTracker.java | 88 ++++
.../hdds/scm/net/TestNetworkTopologyImpl.java | 19 +
.../TestSCMFailoverProxyProviderChangeConfig.java | 149 +++++++
.../hdds/server/http/TestBaseHttpServer.java | 53 +++
.../hadoop/hdds/server/http/TestHttpServer2.java | 390 ++++++++++++++++++
.../hdds/server/http/TestHttpServer2SSL.java | 36 +-
.../hdds/utils/db/TestTableCacheUpdateTracker.java | 118 ++++++
hadoop-hdds/rocks-native/pom.xml | 2 +-
hadoop-hdds/server-scm/pom.xml | 12 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../hdds/scm/ha/SCMHADBTransactionBuffer.java | 2 +
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 41 ++
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 5 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 17 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 24 --
.../hadoop/hdds/scm/node/NodeStateManager.java | 32 +-
.../hdds/scm/node/PendingContainerTracker.java | 2 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 221 ++++++----
.../scm/node/UnhealthyToHealthyNodeHandler.java | 15 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +-
.../scm/pipeline/WritableECContainerProvider.java | 5 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +-
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../scm/ha/TestSCMHADBTransactionBufferImpl.java | 333 +++++++++++++++
.../ha/TestSCMHATransactionBufferMonitorTask.java | 1 +
.../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 124 +++++-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 53 +++
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 57 +--
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 88 +++-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 63 ++-
.../admin/om/BucketDeletedBytesSubCommand.java | 101 +++++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 1 +
.../ozone/client/S3HeadObjectAttributes.java | 10 +-
.../ozone/client/io/OzoneCryptoInputStream.java | 5 +
.../ozone/client/protocol/ClientProtocol.java | 28 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 41 +-
.../hadoop/ozone/client/rpc/TestRpcClient.java | 17 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 3 +
...doopRpcOMFollowerReadFailoverProxyProvider.java | 73 +++-
.../ozone/om/ha/OMFailoverProxyProviderBase.java | 21 +
.../ozone/om/helpers/BucketDeletedBytes.java | 79 ++++
.../ozone/om/helpers/KeyInfoWithVolumeContext.java | 25 +-
.../ozone/om/protocol/OzoneManagerProtocol.java | 8 +
.../ozone/om/protocolPB/GrpcOmTransport.java | 57 ++-
.../ozone/om/protocolPB/Hadoop3OmTransport.java | 4 +-
.../om/protocolPB/OzoneManagerClientProtocol.java | 10 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 86 +++-
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 10 +
...doopRpcOMFollowerReadFailoverProxyProvider.java | 114 +++++-
...OzoneManagerProtocolClientSideTranslatorPB.java | 108 +++++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 119 ++++++
hadoop-ozone/dev-support/checks/integration.sh | 26 +-
hadoop-ozone/dev-support/checks/junit.sh | 3 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 16 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 16 +-
.../src/main/smoketest/s3/secretgenerate.robot | 3 +-
hadoop-ozone/httpfsgateway/pom.xml | 12 +-
.../fs/http/server/HttpFSServerWebServer.java | 4 +
.../main/resources/webapps/webhdfs/WEB-INF/web.xml | 10 +-
.../httpfsgateway/src/main/webapp/WEB-INF/web.xml | 10 +-
hadoop-ozone/iceberg/pom.xml | 24 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 28 +-
hadoop-ozone/integration-test-s3/pom.xml | 40 +-
.../hadoop/ozone/s3/MultiS3GatewayService.java | 21 +-
.../org/apache/hadoop/ozone/s3/ProxyServer.java | 22 +-
.../ozone/s3/ProxyServerIntegrationTest.java | 4 +-
.../apache/hadoop/ozone/s3/S3ClientFactory.java | 1 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 115 +++++-
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 453 ++++++++++++++++++++-
hadoop-ozone/integration-test/pom.xml | 50 +--
.../hdds/scm/TestOmSCMNodesReconfiguration.java | 363 +++++++++++++++++
.../apache/hadoop/hdds/scm/node/TestQueryNode.java | 23 ++
.../hadoop/hdds/scm/storage/TestCommitWatcher.java | 19 +-
.../org/apache/hadoop/ozone/TestCpuMetrics.java | 5 +-
.../client/rpc/TestBlockDataStreamOutput.java | 42 +-
.../rpc/TestContainerStateMachineStream.java | 10 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 127 ++++--
.../rpc/read/TestStreamBlockInputStream.java | 88 ++--
.../ozone/om/OzoneManagerHAFollowerReadTests.java | 8 +-
.../ozone/om/TestGetDBUpdatesAuthorization.java | 73 +++-
...stOzoneManagerHAFollowerReadWithAllRunning.java | 45 +-
...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 4 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 8 +-
.../TestOmMixedWorkloadUnderDeletionBench.java | 19 +-
...TestSnapshotDeletingServiceIntegrationTest.java | 125 +++---
.../hadoop/ozone/shell/TestOzoneShellHA.java | 56 ++-
.../src/main/proto/OmClientProtocol.proto | 35 +-
hadoop-ozone/ozone-manager/pom.xml | 12 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 6 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 35 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 226 ++++++++--
.../apache/hadoop/ozone/om/S3SecretManager.java | 4 +
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 37 ++
.../hadoop/ozone/om/execution/OMExecutionFlow.java | 20 +-
.../hadoop/ozone/om/lock/OMLockDetailsUtil.java | 48 +++
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 7 +-
.../hadoop/ozone/om/ratis/OMThreadContext.java | 198 +++++++++
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 26 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 53 +--
.../hadoop/ozone/om/request/OMClientRequest.java | 26 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 6 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 30 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 3 +-
.../hadoop/ozone/om/response/CleanupTableInfo.java | 51 ---
.../ozone/om/response/DummyOMClientResponse.java | 1 -
.../hadoop/ozone/om/response/OMClientResponse.java | 24 +-
.../om/response/bucket/OMBucketCreateResponse.java | 5 -
.../om/response/bucket/OMBucketDeleteResponse.java | 6 -
.../response/bucket/OMBucketSetOwnerResponse.java | 4 -
.../bucket/OMBucketSetPropertyResponse.java | 4 -
.../response/bucket/acl/OMBucketAclResponse.java | 4 -
.../response/file/OMDirectoryCreateResponse.java | 4 -
.../file/OMDirectoryCreateResponseWithFSO.java | 14 +-
.../om/response/file/OMFileCreateResponse.java | 5 -
.../response/file/OMFileCreateResponseWithFSO.java | 7 -
.../om/response/file/OMRecoverLeaseResponse.java | 5 -
.../om/response/key/OMAllocateBlockResponse.java | 5 -
.../key/OMAllocateBlockResponseWithFSO.java | 5 -
.../key/OMDirectoriesPurgeResponseWithFSO.java | 8 -
.../ozone/om/response/key/OMKeyCommitResponse.java | 8 -
.../response/key/OMKeyCommitResponseWithFSO.java | 8 -
.../ozone/om/response/key/OMKeyCreateResponse.java | 6 -
.../response/key/OMKeyCreateResponseWithFSO.java | 7 -
.../ozone/om/response/key/OMKeyDeleteResponse.java | 7 -
.../response/key/OMKeyDeleteResponseWithFSO.java | 10 -
.../ozone/om/response/key/OMKeyPurgeResponse.java | 4 -
.../ozone/om/response/key/OMKeyRenameResponse.java | 5 -
.../response/key/OMKeyRenameResponseWithFSO.java | 7 -
.../om/response/key/OMKeySetTimesResponse.java | 4 -
.../response/key/OMKeySetTimesResponseWithFSO.java | 5 -
.../om/response/key/OMKeysDeleteResponse.java | 7 -
.../response/key/OMKeysDeleteResponseWithFSO.java | 11 -
.../om/response/key/OMKeysRenameResponse.java | 4 -
.../om/response/key/OMOpenKeysDeleteResponse.java | 8 -
.../om/response/key/acl/OMKeyAclResponse.java | 4 -
.../response/key/acl/OMKeyAclResponseWithFSO.java | 5 -
.../key/acl/prefix/OMPrefixAclResponse.java | 4 -
.../OMLifecycleConfigurationDeleteResponse.java | 4 -
.../OMLifecycleConfigurationSetResponse.java | 4 -
.../OMLifecycleSaveScanStateResponse.java | 3 -
.../OMLifecycleSetServiceStatusResponse.java | 2 -
.../S3ExpiredMultipartUploadsAbortResponse.java | 10 -
.../S3InitiateMultipartUploadResponse.java | 5 -
.../S3InitiateMultipartUploadResponseWithFSO.java | 7 -
.../multipart/S3MultipartUploadAbortResponse.java | 9 -
.../S3MultipartUploadAbortResponseWithFSO.java | 9 -
.../S3MultipartUploadCommitPartResponse.java | 8 -
...S3MultipartUploadCommitPartResponseWithFSO.java | 9 -
.../S3MultipartUploadCompleteResponse.java | 10 -
.../S3MultipartUploadCompleteResponseWithFSO.java | 10 -
.../response/s3/security/OMSetSecretResponse.java | 4 -
.../response/s3/security/S3AssumeRoleResponse.java | 2 -
.../security/S3DeleteRevokedSTSTokensResponse.java | 3 -
.../response/s3/security/S3GetSecretResponse.java | 4 -
.../s3/security/S3RevokeSTSTokenResponse.java | 3 -
.../s3/security/S3RevokeSecretResponse.java | 4 -
.../s3/tagging/S3DeleteObjectTaggingResponse.java | 4 -
.../S3DeleteObjectTaggingResponseWithFSO.java | 4 -
.../s3/tagging/S3PutObjectTaggingResponse.java | 4 -
.../tagging/S3PutObjectTaggingResponseWithFSO.java | 4 -
.../tenant/OMSetRangerServiceVersionResponse.java | 4 -
.../s3/tenant/OMTenantAssignAdminResponse.java | 6 -
.../tenant/OMTenantAssignUserAccessIdResponse.java | 10 -
.../response/s3/tenant/OMTenantCreateResponse.java | 8 -
.../response/s3/tenant/OMTenantDeleteResponse.java | 8 -
.../s3/tenant/OMTenantRevokeAdminResponse.java | 6 -
.../tenant/OMTenantRevokeUserAccessIdResponse.java | 10 -
.../security/OMCancelDelegationTokenResponse.java | 4 -
.../security/OMGetDelegationTokenResponse.java | 4 -
.../security/OMRenewDelegationTokenResponse.java | 4 -
.../snapshot/OMSnapshotCreateResponse.java | 7 -
.../snapshot/OMSnapshotDeleteResponse.java | 4 -
.../OMSnapshotMoveDeletedKeysResponse.java | 3 -
.../snapshot/OMSnapshotMoveTableKeysResponse.java | 3 -
.../response/snapshot/OMSnapshotPurgeResponse.java | 8 -
.../snapshot/OMSnapshotRenameResponse.java | 4 -
.../snapshot/OMSnapshotSetPropertyResponse.java | 4 -
.../upgrade/OMCompleteFinalizeUpgradeResponse.java | 3 -
.../upgrade/OMStartFinalizeUpgradeResponse.java | 4 -
.../om/response/util/OMEchoRPCWriteResponse.java | 2 -
.../om/response/volume/OMQuotaRepairResponse.java | 5 -
.../om/response/volume/OMVolumeAclOpResponse.java | 4 -
.../om/response/volume/OMVolumeCreateResponse.java | 4 -
.../om/response/volume/OMVolumeDeleteResponse.java | 4 -
.../response/volume/OMVolumeSetOwnerResponse.java | 4 -
.../response/volume/OMVolumeSetQuotaResponse.java | 4 -
.../ozone/om/service/KeyLifecycleService.java | 10 +-
.../hadoop/ozone/om/service/QuotaRepairTask.java | 129 +++++-
.../ozone/om/snapshot/SnapshotDiffManager.java | 15 +-
.../trapped/BucketDeletedDataCalculator.java | 373 +++++++++++++++++
.../ozone/om/snapshot/trapped}/package-info.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 46 ++-
.../protocolPB/OzoneManagerRequestHandler.java | 27 +-
.../hadoop/ozone/om/TestEDEKCacheLoader.java | 78 ++++
.../ozone/om/TestOmSnapshotLeakDetection.java | 87 ++++
.../org/apache/hadoop/ozone/om/TestScmClient.java | 24 ++
.../ozone/om/execution/TestOMExecutionFlow.java | 165 ++++++++
.../ozone/om/lock/TestOMLockDetailsUtil.java | 78 ++++
.../hadoop/ozone/om/ratis/TestOMThreadContext.java | 207 ++++++++++
...tOzoneManagerDoubleBufferWithDummyResponse.java | 74 +++-
.../om/ratis/TestOzoneManagerStateMachine.java | 226 ++++++++++
.../om/request/key/TestOMKeyCreateRequest.java | 9 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 7 +
.../ozone/om/response/TestCleanupTableInfo.java | 384 -----------------
.../ozone/om/response/TestOMClientResponse.java} | 42 +-
.../hadoop/ozone/om/response/TestOMResponse.java | 51 ++-
.../hadoop/ozone/om/response/TestS3DerivedKey.java | 241 +++++++++++
.../file/TestOMDirectoryCreateResponseWithFSO.java | 2 -
.../ozone/om/service/TestKeyLifecycleService.java | 10 +-
.../ozone/om/service/TestQuotaRepairTask.java | 425 +++++++++++++++++++
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 18 +
.../trapped/TestBucketDeletedDataCalculator.java | 243 +++++++++++
hadoop-ozone/recon/pom.xml | 12 +-
.../ozone/recon/api/ClusterStateEndpoint.java | 2 +-
.../ozone/recon/api/filters/ReconAuthFilter.java | 2 +-
.../webapps/recon/ozone-recon-web/package.json | 4 +-
.../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 34 +-
.../src/v2/components/navBar/navBar.tsx | 12 +-
.../src/v2/pages/heatmap/heatmap.tsx | 35 +-
hadoop-ozone/s3gateway/pom.xml | 26 +-
.../org/apache/hadoop/ozone/audit/S3GAction.java | 6 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 3 +-
.../ReadConsistencyFilter.java} | 40 +-
.../hadoop/ozone/s3/S3ContentTypeFilter.java | 135 ++++++
.../hadoop/ozone/s3/S3GatewayHttpServer.java | 13 +
.../hadoop/ozone/s3/S3GatewayWebAdminServer.java | 9 +-
.../apache/hadoop/ozone/s3/S3STSHttpServer.java | 3 +
.../hadoop/ozone/s3/SignedChunksInputStream.java | 152 ++++++-
.../apache/hadoop/ozone/s3/WeldCdiIntegration.java | 50 +++
.../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 5 +
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 6 +
.../s3/endpoint/BucketGetLocationHandler.java | 5 +
.../ozone/s3/endpoint/BucketLifecycleHandler.java | 9 +-
.../ozone/s3/endpoint/BucketOperationHandler.java | 21 +-
.../BucketOperationNotImplementedHandler.java | 64 +++
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 59 ++-
.../s3/endpoint/ListMultipartUploadsHandler.java | 15 +-
.../ozone/s3/endpoint/MultipartKeyHandler.java | 12 +-
.../hadoop/ozone/s3/endpoint/ObjectAclHandler.java | 15 +-
.../ozone/s3/endpoint/ObjectAttributesHandler.java | 43 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 16 +-
.../ozone/s3/endpoint/ObjectGetTorrentHandler.java | 5 +
.../ozone/s3/endpoint/ObjectOperationHandler.java | 17 +-
.../ObjectOperationNotImplementedHandler.java | 65 +++
.../ozone/s3/endpoint/ObjectTaggingHandler.java | 19 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 8 +-
.../hadoop/ozone/s3/endpoint/S3RequestContext.java | 12 +
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 9 +
.../ozone/s3/signature/AWSSignatureProcessor.java | 3 +-
.../s3/signature/AuthorizationV4HeaderParser.java | 43 +-
.../s3/signature/AuthorizationV4QueryParser.java | 7 +-
.../hadoop/ozone/s3/signature/ChunksValidator.java | 39 +-
.../hadoop/ozone/s3/signature/Credential.java | 20 +
.../s3/signature/MalformedResourceException.java | 19 +-
.../ozone/s3/signature/StringToSignProducer.java | 50 ---
.../ozone/s3/util/ReadConsistencyContext.java | 112 +++++
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 45 ++
.../hadoop/ozone/s3/util/S3GActionIamMapper.java | 2 +
.../ozone/s3secret/S3SecretManagementEndpoint.java | 17 +-
.../resources/webapps/s3gateway/WEB-INF/web.xml | 8 +
.../hadoop/ozone/client/ClientProtocolStub.java | 42 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 2 +-
.../hadoop/ozone/s3/TestAuthorizationFilter.java | 94 +++--
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 58 +++
.../hadoop/ozone/s3/TestReadConsistencyFilter.java | 51 +++
.../hadoop/ozone/s3/TestS3ContentTypeFilter.java | 127 ++++++
.../hadoop/ozone/s3/TestS3GatewayHttpServers.java | 100 +++++
.../ozone/s3/TestSignedChunksInputStream.java | 173 ++++++++
.../ozone/s3/endpoint/EndpointTestUtils.java | 20 +
.../hadoop/ozone/s3/endpoint/TestBucketList.java | 389 ++++++++----------
.../s3/endpoint/TestBucketNotImplemented.java | 102 +++++
.../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 189 +++++++++
.../s3/endpoint/TestListMultipartUploads.java | 166 ++++++++
.../hadoop/ozone/s3/endpoint/TestListParts.java | 108 +++++
.../ozone/s3/endpoint/TestObjectAttributesGet.java | 146 ++++++-
.../ozone/s3/endpoint/TestObjectMultiDelete.java | 24 ++
.../s3/endpoint/TestObjectNotImplemented.java | 113 +++++
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 130 +++++-
.../ozone/s3/endpoint/TestObjectTaggingDelete.java | 39 +-
.../ozone/s3/endpoint/TestObjectTaggingGet.java | 20 +
.../ozone/s3/endpoint/TestObjectTaggingPut.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 39 ++
.../ozone/s3/endpoint/TestPermissionCheck.java | 10 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 111 ++++-
.../TestS3LifecycleConfigurationDelete.java | 125 ++++++
.../endpoint/TestS3LifecycleConfigurationGet.java | 90 ++++
.../endpoint/TestS3LifecycleConfigurationPut.java | 90 ++++
.../ozone/s3/endpoint/TestS3RequestContext.java | 86 ++++
.../ozone/s3/endpoint/TestUploadWithStream.java | 34 ++
.../ozone/s3/signature/SignatureTestUtils.java | 34 ++
.../signature/TestAuthorizationV4HeaderParser.java | 133 +++---
.../signature/TestAuthorizationV4QueryParser.java | 55 ++-
.../ozone/s3/signature/TestChunksValidator.java | 56 +++
.../s3/signature/TestStringToSignProducer.java | 56 +--
.../ozone/s3/util/TestS3GActionIamMapper.java | 2 +
.../hadoop/ozone/s3secret/TestSecretGenerate.java | 6 +-
.../hadoop/ozone/s3secret/TestSecretRevoke.java | 5 +
pom.xml | 77 ++--
350 files changed, 12741 insertions(+), 2677 deletions(-)
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/TestGrpcXceiverService.java
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/{AdminAuthorizedServlet.java
=> HddsJMXJsonServlet.java} (52%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/cache/TableCacheUpdateTracker.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/scm/proxy/TestSCMFailoverProxyProviderChangeConfig.java
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestTableCacheUpdateTracker.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHADBTransactionBufferImpl.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/BucketDeletedBytesSubCommand.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestOmSCMNodesReconfiguration.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OMLockDetailsUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OMThreadContext.java
delete mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/CleanupTableInfo.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java
copy {hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage =>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped}/package-info.java
(85%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestEDEKCacheLoader.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLeakDetection.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/execution/TestOMExecutionFlow.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestOMLockDetailsUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOMThreadContext.java
delete mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java
copy
hadoop-ozone/ozone-manager/src/{main/java/org/apache/hadoop/ozone/om/request/upgrade/OMStartFinalizeUpgradeRequest.java
=> test/java/org/apache/hadoop/ozone/om/response/TestOMClientResponse.java}
(52%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestS3DerivedKey.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java
copy
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/{s3sts/S3STSEnabledEndpointRequestFilter.java
=> s3/ReadConsistencyFilter.java} (51%)
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3ContentTypeFilter.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/WeldCdiIntegration.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketOperationNotImplementedHandler.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectOperationNotImplementedHandler.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/ReadConsistencyContext.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestReadConsistencyFilter.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3ContentTypeFilter.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayHttpServers.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketNotImplemented.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListMultipartUploads.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectNotImplemented.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3RequestContext.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
