This is an automated email from the ASF dual-hosted git repository. devmadhuu pushed a commit to branch HDDS-11233 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 6a58b98429953b81dc4457a6b2c37a0f1f9c2a59 Merge: 8787bc21038 96cb1a6ca40 Author: Devesh Singh <[email protected]> AuthorDate: Mon Oct 12 10:27:22 2026 +0530 HDDS-16116. Merge master changes into HDDS-11233 .github/ci.md | 1 + .github/dependabot.yml | 23 +- .github/labeler.yml | 90 + .github/workflows/asf-allowlist-check.yaml | 2 +- .github/workflows/build-ratis.yml | 4 +- .github/workflows/check.yml | 12 +- .github/workflows/ci-with-ratis.yml | 4 +- .github/workflows/ci.yml | 81 +- .github/workflows/intermittent-test-check.yml | 8 +- .github/workflows/label-pr.yml | 39 +- .github/workflows/populate-cache.yml | 15 +- .github/workflows/post-commit.yml | 2 +- .github/workflows/repeat-acceptance.yml | 4 +- .github/workflows/schedule-label-pr.yml | 7 +- .github/workflows/scheduled-cache-update.yml | 2 +- .github/workflows/zizmor.yml | 4 +- .gitignore | 1 + .mvn/extensions.xml | 2 +- CONTRIBUTING.md | 4 +- dev-support/ci/errorprone.bats | 102 + .../ci/junit_summary.bats | 20 +- dev-support/ci/junit_summary.py | 189 ++ dev-support/ci/selective_ci_checks.bats | 78 +- dev-support/ci/selective_ci_checks.sh | 88 +- dev-support/ci/test_junit_summary.py | 226 ++ .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 23 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 69 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 4 +- .../hadoop/hdds/scm/XceiverClientShortCircuit.java | 111 +- .../hdds/scm/storage/BlockDataStreamOutput.java | 84 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 152 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 17 +- .../hadoop/hdds/scm/storage/ChunkInputStream.java | 177 +- .../hdds/scm/storage/DomainSocketFactory.java | 4 + .../hdds/scm/storage/LocalChunkInputStream.java | 48 +- .../hdds/scm/storage/MultipartInputStream.java | 63 +- .../hadoop/hdds/scm/storage/PartInputStream.java | 13 + .../hdds/scm/storage/StreamBlockInputStream.java | 182 +- .../io/ECBlockReconstructedStripeInputStream.java | 30 +- .../hadoop/hdds/scm/TestOzoneClientConfig.java | 10 +- .../hdds/scm/storage/DummyChunkInputStream.java | 22 +- .../hdds/scm/storage/MockDatanodePipeline.java | 45 +- .../hdds/scm/storage/PositionedReadTestHelper.java | 92 + .../scm/storage/TestBlockDataStreamOutput.java | 92 +- .../hdds/scm/storage/TestBlockInputStream.java | 109 + .../hdds/scm/storage/TestChunkInputStream.java | 71 + .../hdds/scm/storage/TestMultipartInputStream.java | 195 ++ .../scm/storage/TestStreamBlockInputStream.java | 361 +++ .../ozone/client/io/TestECBlockInputStream.java | 2 +- .../TestECBlockReconstructedStripeInputStream.java | 42 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +- .../org/apache/hadoop/hdds/client/BlockID.java | 29 +- .../hadoop/hdds/conf/DelegatingProperties.java | 2 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 42 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 34 + .../hadoop/hdds/scm/StreamingReadResponse.java | 52 + .../scm/container/ReplicationManagerReport.java | 10 - .../balancer/ContainerBalancerAdvisor.java | 438 +++ .../balancer/ContainerBalancerClusterAnalyzer.java | 293 ++ .../balancer/ContainerBalancerClusterSnapshot.java | 250 ++ .../balancer/ContainerBalancerConfiguration.java | 17 +- .../balancer/ContainerBalancerEstimation.java | 205 ++ .../balancer/ContainerBalancerProfile.java | 53 + .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 39 +- .../hdds/scm/storage/ContainerProtocolCalls.java | 105 +- .../apache/hadoop/hdds/tracing/TraceAllMethod.java | 28 +- .../apache/hadoop/hdds/tracing/TracingUtil.java | 4 + .../hadoop/hdds/utils/ConnectionFailureUtils.java | 2 +- .../apache/hadoop/hdds/utils/db/CodecBuffer.java | 9 + .../apache/hadoop/hdds/utils/db/Proto2Codec.java | 7 +- .../hadoop/hdds/utils/db/StringCodecBase.java | 9 +- .../hadoop/io_/retry/RetryInvocationHandler.java | 3 + .../main/java/org/apache/hadoop/ipc_/Client.java | 33 +- .../src/main/java/org/apache/hadoop/ipc_/RPC.java | 3 + .../apache/hadoop/ipc_/RpcInvocationHandler.java | 11 +- .../java/org/apache/hadoop/ipc_/RpcProxy.java} | 16 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../apache/hadoop/ozone/OzoneManagerVersion.java | 7 +- .../org/apache/hadoop/ozone/common/Checksum.java | 68 + .../apache/hadoop/ozone/common/ChecksumCache.java | 10 +- .../apache/hadoop/ozone/common/ChunkBuffer.java | 30 + .../common/ChunkBufferImplWithByteBuffer.java | 12 + .../common/ChunkBufferImplWithByteBufferList.java | 31 +- .../ozone/common/IncrementalChunkBuffer.java | 29 +- .../hadoop/ozone/util/ConcurrentMutableRate.java | 49 + .../org/apache/hadoop/ozone/util/MetricUtil.java | 6 +- .../common/src/main/resources/ozone-default.xml | 174 +- .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 16 + .../hadoop/hdds/conf/TestOzoneConfiguration.java | 83 + .../hadoop/hdds/scm/TestStreamingReadResponse.java | 172 ++ .../balancer/TestContainerBalancerAdvisor.java | 296 ++ .../TestContainerBalancerClusterAnalyzer.java | 296 ++ .../hadoop/hdds/scm/pipeline/TestPipeline.java | 12 + .../hdds/utils/TestConnectionFailureUtils.java | 14 + .../apache/hadoop/hdds/utils/db/CodecTestUtil.java | 6 + .../hadoop/hdds/utils/db/Proto2CodecTestBase.java | 13 + .../apache/hadoop/io_/retry/TestRetryProxy.java | 23 + .../ozone/common/ChunkBufferPutBenchmark.java | 407 +++ .../ozone/common/JfrByteBufferAllocations.java | 133 + .../apache/hadoop/ozone/common/TestChecksum.java | 16 + .../hadoop/ozone/common/TestChecksumCache.java | 33 + .../hadoop/ozone/common/TestChunkBuffer.java | 271 ++ hadoop-hdds/config/pom.xml | 4 + .../hadoop/hdds/conf/ConfigurationSource.java | 10 +- .../hdds/conf/MutableConfigurationSource.java | 67 + .../org/apache/hadoop/hdds/conf/StorageSize.java | 2 +- .../hadoop/hdds/conf/TestConfigurationSource.java | 8 + .../apache/hadoop/hdds/conf/TestStorageSize.java | 65 + .../apache/hadoop/ozone/HddsDatanodeService.java | 26 +- .../common/impl/BlockDeletingService.java | 13 +- .../common/impl/ContainerLayoutVersion.java | 22 +- .../container/common/impl/HddsDispatcher.java | 2 +- .../ozone/container/common/interfaces/Handler.java | 2 +- .../common/statemachine/DatanodeConfiguration.java | 49 + .../common/statemachine/DatanodeStateMachine.java | 12 + .../common/statemachine/EndpointStateMachine.java | 4 +- .../ClosePipelineCommandHandler.java | 4 +- .../states/datanode/RunningDatanodeState.java | 19 +- .../states/endpoint/VersionEndpointTask.java | 31 +- .../transport/server/GrpcXceiverService.java | 68 +- .../common/transport/server/XceiverServerGrpc.java | 5 +- .../server/ratis/ContainerStateMachine.java | 31 +- .../transport/server/ratis/DispatcherContext.java | 4 +- .../common/transport/server/ratis/LocalStream.java | 22 +- .../container/common/utils/ContainerCache.java | 4 +- .../container/common/utils/ReferenceCountedDB.java | 2 +- .../ozone/container/keyvalue/KeyValueHandler.java | 90 +- .../container/keyvalue/ReadBlockComputation.java | 159 ++ .../ozone/container/metadata/DatanodeTable.java | 5 + .../ozone/container/ozoneimpl/OzoneContainer.java | 116 +- .../container/replication/ReplicationServer.java | 4 +- .../checksum/ContainerMerkleTreeTestUtils.java | 34 +- .../container/checksum/TestContainerDiff.java | 19 +- .../container/common/TestBlockDeletingService.java | 154 +- .../container/common/TestDatanodeStateMachine.java | 137 + .../container/common/impl/TestHddsDispatcher.java | 5 +- .../states/datanode/TestRunningDatanodeState.java | 127 +- .../transport/server/TestGrpcXceiverService.java | 169 ++ .../keyvalue/ContainerLayoutTestInfo.java | 90 +- ...ruptions.java => ContainerTestCorruptions.java} | 12 +- ...ava => KeyValueContainerIntegrityTestBase.java} | 4 +- ...stContainerReconciliationWithMockDatanodes.java | 26 +- .../container/keyvalue/TestKeyValueContainer.java | 2 +- .../keyvalue/TestKeyValueContainerCheck.java | 24 +- .../TestKeyValueContainerMetadataInspector.java | 2 +- .../container/keyvalue/TestKeyValueHandler.java | 374 ++- .../container/ozoneimpl/TestOzoneContainer.java | 81 + hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 6 + .../docs/content/design/efficient-snapdiff.md | 9 +- hadoop-hdds/docs/content/design/ozone-sts.md | 56 +- .../docs/content/feature/Reconfigurability.md | 4 +- hadoop-hdds/docs/content/feature/SCM-HA.md | 3 +- hadoop-hdds/docs/content/interface/S3.md | 24 + hadoop-hdds/framework/pom.xml | 24 +- .../SCMSecurityProtocolClientSideTranslatorPB.java | 5 +- .../SecretKeyProtocolClientSideTranslatorPB.java | 2 +- .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 4 +- ...lockLocationProtocolClientSideTranslatorPB.java | 2 +- .../scm/proxy/SCMFailoverProxyProviderBase.java | 91 +- .../certificate/client/SCMCertificateClient.java | 2 +- .../org/apache/hadoop/hdds/server/ServerUtils.java | 3 +- .../hdds/server/http/AdminAuthorizedServlet.java | 2 +- .../hadoop/hdds/server/http/BaseHttpServer.java | 80 +- ...horizedServlet.java => HddsJMXJsonServlet.java} | 28 +- .../hadoop/hdds/server/http/HttpServer2.java | 258 +- .../hdds/server/http/ServletElementsFactory.java | 4 +- .../hadoop/hdds/utils/BackgroundService.java | 14 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 27 +- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 48 +- .../org/apache/hadoop/hdds/utils/NettyMetrics.java | 108 +- .../apache/hadoop/hdds/utils/ScmNodeAddress.java | 31 +- .../hadoop/hdds/utils/db/RDBCheckpointManager.java | 8 +- .../org/apache/hadoop/hdds/utils/db/RDBTable.java | 35 +- .../apache/hadoop/hdds/utils/db/RocksDatabase.java | 84 +- .../org/apache/hadoop/hdds/utils/db/Table.java | 22 + .../apache/hadoop/hdds/utils/db/TypedTable.java | 150 + .../utils/db/cache/TableCacheUpdateTracker.java | 88 + .../exceptions/TestSCMExceptionResultCodes.java | 3 +- .../hdds/scm/net/TestNetworkTopologyImpl.java | 19 + .../TestSCMFailoverProxyProviderChangeConfig.java | 149 + .../TestSCMSecurityExceptionErrorCodes.java} | 34 +- .../apache/hadoop/hdds/server/TestServerUtils.java | 16 + .../hdds/server/http/TestBaseHttpServer.java | 83 + .../hadoop/hdds/server/http/TestHttpServer2.java | 494 ++++ .../hdds/server/http/TestHttpServer2SSL.java | 62 +- .../apache/hadoop/hdds/utils/TestNettyMetrics.java | 89 + .../hadoop/hdds/utils/db/InMemoryTestTable.java | 145 +- .../org/apache/hadoop/hdds/utils/db/TestCodec.java | 14 + .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 46 + .../apache/hadoop/hdds/utils/db/TestRDBTable.java | 23 + .../hadoop/hdds/utils/db/TestRDBTableStore.java | 132 + .../hdds/utils/db/TestTableCacheUpdateTracker.java | 118 + hadoop-hdds/hadoop-dependency-client/pom.xml | 4 + hadoop-hdds/interface-admin/pom.xml | 1 + hadoop-hdds/interface-client/pom.xml | 1 + hadoop-hdds/interface-server/pom.xml | 1 + hadoop-hdds/rocks-native/pom.xml | 2 +- ...wSstFileRecords.java => RawSstFileRecords.java} | 4 +- ...estLatestVersionedKWayMergeIteratorOverSst.java | 4 +- .../org/apache/ozone/rocksdiff/CompactionDag.java | 2 +- .../apache/ozone/rocksdiff/TestCompactionDag.java | 19 + .../rocksdiff/TestRocksDBCheckpointDiffer.java | 169 +- .../apache/ozone/rocksdiff/TestRocksDiffUtils.java | 18 +- hadoop-hdds/server-scm/pom.xml | 12 +- .../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 33 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +- .../scm/block/DeletedBlockLogStateManagerImpl.java | 27 +- .../scm/container/balancer/ContainerBalancer.java | 18 +- .../ContainerBalancerSelectionCriteria.java | 22 +- .../container/balancer/ContainerBalancerTask.java | 48 +- .../replication/ContainerCheckRequest.java | 11 + .../container/replication/ReplicationManager.java | 6 +- .../replication/ReplicationManagerMetrics.java | 37 +- .../health/ClosedWithUnhealthyReplicasHandler.java | 2 +- .../health/ClosingContainerHandler.java | 2 +- .../health/ECMisReplicationCheckHandler.java | 4 +- .../health/ECReplicationCheckHandler.java | 7 +- .../replication/health/EmptyContainerHandler.java | 6 +- .../replication/health/OpenContainerHandler.java | 8 +- .../health/QuasiClosedContainerHandler.java | 2 +- .../health/QuasiClosedStuckReplicationCheck.java | 9 +- .../health/RatisReplicationCheckHandler.java | 10 +- .../RatisUnhealthyReplicationCheckHandler.java | 8 +- .../health/VulnerableUnhealthyReplicasHandler.java | 4 +- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 12 +- .../hdds/scm/ha/InterSCMGrpcProtocolService.java | 23 +- .../hdds/scm/ha/SCMHADBTransactionBuffer.java | 6 + .../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 51 + .../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 15 + .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 8 +- .../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 7 + .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 4 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 + .../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 17 +- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 24 - .../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 15 - .../hdds/scm/node/NodeDecommissionManager.java | 6 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 32 +- .../hdds/scm/node/PendingContainerTracker.java | 2 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 244 +- .../hadoop/hdds/scm/node/states/DatanodeEntry.java | 6 +- .../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +- .../scm/pipeline/WritableECContainerProvider.java | 5 +- .../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +- .../hdds/scm/server/SCMClientProtocolServer.java | 24 +- .../hdds/scm/server/StorageContainerManager.java | 5 +- .../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 39 + .../org/apache/hadoop/hdds/scm/TestScmUtils.java | 63 + .../container/balancer/TestContainerBalancer.java | 50 + .../TestContainerBalancerDatanodeNodeLimit.java | 88 + .../TestContainerBalancerSelectionCriteria.java | 64 + .../replication/TestContainerCheckRequest.java | 58 + .../TestQuasiClosedStuckReplicationCheck.java | 1 - .../hadoop/hdds/scm/ha/TestInterSCMGrpcClient.java | 119 + .../scm/ha/TestSCMHADBTransactionBufferImpl.java | 333 +++ .../ha/TestSCMHATransactionBufferMonitorTask.java | 209 +- .../hadoop/hdds/scm/ha/TestSCMRatisServerImpl.java | 93 +- .../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 115 + .../hdds/scm/ha/TestSequenceIDGenerator.java | 53 + .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 181 +- .../hadoop/hdds/scm/node/TestNodeStateManager.java | 85 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 223 +- .../hdds/scm/server/TestSCMContainerMetrics.java | 21 +- .../TestStorageContainerManagerRatisRoles.java | 126 + hadoop-hdds/test-utils/pom.xml | 4 + .../org/apache/ozone/test/GenericTestUtils.java | 7 +- .../cli/ContainerBalancerAssessmentSubcommand.java | 316 +++ .../hdds/scm/cli/ContainerBalancerCommands.java | 38 +- ...nd.java => ContainerBalancerConfigOptions.java} | 149 +- .../cli/ContainerBalancerEstimateSubcommand.java | 175 ++ .../scm/cli/ContainerBalancerStartSubcommand.java | 92 +- .../hdds/scm/cli/SafeModeCheckSubcommand.java | 10 +- .../admin/om/BucketDeletedBytesSubCommand.java | 101 + .../ozone/admin/om/GetServiceRolesSubcommand.java | 8 +- .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 1 + .../admin/scm/TransferScmLeaderSubCommand.java | 2 +- .../TestContainerBalancerAssessmentSubcommand.java | 176 ++ .../TestContainerBalancerSubCommand.java | 264 +- .../hdds/scm/cli/TestSafeModeCheckSubcommand.java | 123 + .../ozone/scm/TestGetScmRatisRolesSubcommand.java | 85 +- .../debug/replicas/BlockExistenceVerifier.java | 2 +- .../debug/replicas/chunk/ChunkKeyHandler.java | 2 +- .../java/org/apache/ozone/TestPrintableGraph.java | 11 +- .../schemaupgrade/UpgradeContainerSchema.java | 4 +- .../schemaupgrade/TestUpgradeContainerSchema.java | 9 + .../ozone/shell/s3/RevokeSTSTokenHandler.java | 73 + .../org/apache/hadoop/ozone/shell/s3/S3Shell.java | 3 +- .../ozone/shell/snapshot/CountSnapshotHandler.java | 91 + .../ozone/shell/snapshot/SnapshotCommands.java | 1 + .../shell/snapshot/TestCountSnapshotHandler.java | 122 + .../apache/hadoop/ozone/client/ObjectStore.java | 47 + .../apache/hadoop/ozone/client/OzoneBucket.java | 51 +- .../ozone/client/S3HeadObjectAttributes.java | 55 + .../client/checksum/ECBlockChecksumComputer.java | 2 +- .../client/checksum/ECFileChecksumHelper.java | 2 +- .../checksum/ReplicatedFileChecksumHelper.java | 2 +- .../client/io/BlockDataStreamOutputEntryPool.java | 8 +- .../client/io/BlockOutputStreamEntryPool.java | 14 +- .../hadoop/ozone/client/io/ECKeyOutputStream.java | 5 +- .../ozone/client/io/KeyDataStreamOutput.java | 4 + .../hadoop/ozone/client/io/KeyOutputStream.java | 4 + .../ozone/client/io/OzoneCryptoInputStream.java | 164 +- .../ozone/client/io/OzoneDataStreamOutput.java | 10 +- .../hadoop/ozone/client/io/OzoneOutputStream.java | 10 +- .../ozone/client/protocol/ClientProtocol.java | 111 +- .../client/protocol/ListStatusLightOptions.java | 182 ++ .../apache/hadoop/ozone/client/rpc/RpcClient.java | 123 +- .../hadoop/ozone/client/TestOzoneECClient.java | 21 + .../checksum/TestECBlockChecksumComputer.java | 122 + .../ozone/client/io/TestKeyDataStreamOutput.java | 2 +- .../ozone/client/io/TestKeyOutputStream.java | 76 + .../client/io/TestOzoneCryptoInputStream.java | 504 ++++ .../hadoop/ozone/client/rpc/TestRpcClient.java | 17 + hadoop-ozone/common/pom.xml | 4 + .../main/java/org/apache/hadoop/ozone/OmUtils.java | 30 +- .../ozone/conf/S3GatewayHealthCheckConfig.java | 95 + .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 39 +- .../hadoop/ozone/om/exceptions/OMException.java | 4 +- ...doopRpcOMFollowerReadFailoverProxyProvider.java | 245 +- .../ozone/om/ha/OMFailoverProxyProviderBase.java | 21 + .../ozone/om/helpers/AssumeRoleResponseInfo.java | 109 + .../ozone/om/helpers/AwsRoleArnValidator.java | 146 + .../ozone/om/helpers/BucketDeletedBytes.java | 79 + .../ozone/om/helpers/CallerIdentityInfo.java | 88 + .../ozone/om/helpers/KeyInfoWithVolumeContext.java | 25 +- .../hadoop/ozone/om/helpers/OMRatisHelper.java | 11 +- .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 19 + .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 13 + .../helpers/OmMultipartCommitUploadPartInfo.java | 9 +- .../hadoop/ozone/om/helpers/OmRangerSyncArgs.java | 3 - .../apache/hadoop/ozone/om/helpers/S3STSUtils.java | 226 ++ .../ozone/om/protocol/OzoneManagerProtocol.java | 64 +- .../apache/hadoop/ozone/om/protocol/S3Auth.java | 20 + .../ozone/om/protocolPB/GrpcOmTransport.java | 57 +- .../ozone/om/protocolPB/Hadoop3OmTransport.java | 4 +- .../om/protocolPB/OzoneManagerClientProtocol.java | 10 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 212 +- .../om/protocolPB/OzoneManagerProtocolPB.java | 3 +- .../security/acl/iam/IamSessionPolicyResolver.java | 1231 ++++++++ .../ozone/security/acl/iam/package-info.java | 8 +- .../SnapshotBucketCount.java} | 41 +- .../SnapshotCountResponse.java} | 37 +- .../java/org/apache/hadoop/ozone/TestOmUtils.java | 80 + .../ozone/om/exceptions/TestResultCodes.java | 2 +- ...doopRpcOMFollowerReadFailoverProxyProvider.java | 236 +- .../ozone/om/ha/TestOMFailoverProxyProvider.java | 17 + .../om/helpers/TestAssumeRoleResponseInfo.java | 197 ++ .../ozone/om/helpers/TestAwsRoleArnValidator.java | 133 + .../ozone/om/helpers/TestCallerIdentityInfo.java | 107 + .../hadoop/ozone/om/helpers/TestOMRatisHelper.java | 106 + .../hadoop/ozone/om/helpers/TestS3STSUtils.java | 98 + .../om/helpers/TestS3STSUtilsCallerIdentity.java | 59 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 171 ++ .../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 119 + .../grpc/TestClientAddressClientInterceptor.java | 1 + .../grpc/TestClientAddressServerInterceptor.java | 14 +- .../acl/iam/TestIamSessionPolicyResolver.java | 2927 ++++++++++++++++++++ .../ozone/security/acl/iam/package-info.java | 8 +- hadoop-ozone/dev-support/checks/bats.sh | 2 + hadoop-ozone/dev-support/checks/errorprone.sh | 62 + hadoop-ozone/dev-support/checks/findbugs.sh | 2 +- hadoop-ozone/dev-support/checks/integration.sh | 26 +- hadoop-ozone/dev-support/checks/junit.sh | 3 +- hadoop-ozone/dev-support/checks/license.sh | 1 + hadoop-ozone/dist/pom.xml | 22 +- .../Ozone - Memory Consumption Metrics.json | 4 +- .../dist/src/main/compose/common/init-kdc.sh | 3 + .../dist/src/main/compose/common/ranger.yaml | 16 +- .../dist/src/main/compose/common/s3-haproxy.cfg | 11 +- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 + .../dist/src/main/compose/ozonesecure-ha/.env | 1 + .../src/main/compose/ozonesecure-ha/docker-config | 3 + .../main/compose/ozonesecure-ha/polaris-setup.sh | 121 + .../compose/ozonesecure-ha/polaris-smoketest.sh | 112 + .../src/main/compose/ozonesecure-ha/polaris.yaml | 77 + .../main/compose/ozonesecure-ha/ranger-testlib.sh | 111 + .../src/main/compose/ozonesecure-ha/ranger.yaml | 6 +- .../src/main/compose/ozonesecure-ha/s3-haproxy.cfg | 11 +- .../src/main/compose/ozonesecure-ha/test-ranger.sh | 50 +- .../compose/ozonesecure-ha/test-repair-tools.sh | 67 +- .../compose/ozonesecure-ha/{.env => test-sts.sh} | 28 +- .../src/main/compose/ozonesecure/docker-config | 1 + hadoop-ozone/dist/src/main/compose/testlib.sh | 13 + hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 2 +- .../dist/src/main/compose/upgrade/testlib.sh | 3 +- .../dist/src/main/compose/xcompat/clients.yaml | 8 + hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 2 +- .../src/main/k8s/definitions/ozone/s3g-ss.yaml | 6 +- .../examples/getting-started/s3g-statefulset.yaml | 6 +- .../k8s/examples/minikube/s3g-statefulset.yaml | 6 +- .../k8s/examples/ozone-dev/s3g-statefulset.yaml | 6 +- .../k8s/examples/ozone-ha/s3g-statefulset.yaml | 6 +- .../main/k8s/examples/ozone/s3g-statefulset.yaml | 6 +- hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 24 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 20 +- .../smoketest/repair/om-update-transaction.robot | 56 + .../dist/src/main/smoketest/s3/bucketcreate.robot | 55 + .../dist/src/main/smoketest/s3/health.robot | 43 + .../src/main/smoketest/s3/secretgenerate.robot | 3 +- .../security/generate_oversized_session_policy.py} | 34 +- .../smoketest/security/mutate_sts_session_token.py | 216 ++ .../security/ozone-secure-sts-multitenant.robot | 336 +++ .../security/ozone-secure-sts-polaris.sql | 27 + .../smoketest/security/ozone-secure-sts.resource | 376 +++ .../main/smoketest/security/ozone-secure-sts.robot | 1474 ++++++++++ .../smoketest/security/ozone-secure-token.robot | 11 + hadoop-ozone/freon/pom.xml | 1 + .../ozone/freon/HadoopFsReadWriteValidator.java | 263 ++ hadoop-ozone/httpfsgateway/pom.xml | 12 +- .../fs/http/server/HttpFSExceptionProvider.java | 5 +- .../apache/ozone/fs/http/server/HttpFSServer.java | 2 +- .../fs/http/server/HttpFSServerWebServer.java | 4 + .../main/resources/webapps/webhdfs/WEB-INF/web.xml | 10 +- .../httpfsgateway/src/main/webapp/WEB-INF/web.xml | 10 +- .../http/server/TestHttpFSExceptionProvider.java | 59 + hadoop-ozone/iceberg/pom.xml | 26 +- hadoop-ozone/insight/pom.xml | 3 +- .../hadoop/ozone/recon/TestReconAsPassiveScm.java | 28 +- .../src/test/resources/ozone-site.xml | 77 +- hadoop-ozone/integration-test-s3/pom.xml | 40 +- .../hadoop/ozone/s3/MultiS3GatewayService.java | 21 +- .../org/apache/hadoop/ozone/s3/ProxyServer.java | 50 +- .../ozone/s3/ProxyServerIntegrationTest.java | 4 +- .../apache/hadoop/ozone/s3/S3ClientFactory.java | 1 + .../hadoop/ozone/s3/TestS3GatewayHealthCheck.java | 126 + .../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 33 + .../hadoop/ozone/s3/awssdk/package-info.java | 8 +- .../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 222 +- .../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 689 ++++- .../hadoop/ozone/s3/awssdk/v2/package-info.java | 8 +- .../src/test/resources/ozone-site.xml | 86 +- hadoop-ozone/integration-test/pom.xml | 50 +- .../fs/contract/AbstractContractSafeModeTest.java | 2 +- .../ozone/AbstractRootedOzoneFileSystemTest.java | 66 +- .../java/org/apache/hadoop/fs/ozone/TestHSync.java | 26 +- .../fs/ozone/TestOfsGetFileStatusBenchmark.java | 356 +++ .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 8 +- .../hadoop/hdds/scm/TestContainerOperations.java | 25 +- .../hdds/scm/TestOmSCMNodesReconfiguration.java | 363 +++ .../hadoop/hdds/scm/TestRackAwarePlacement.java | 4 + .../hadoop/hdds/scm/TestSecretKeySnapshot.java | 104 +- .../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 113 +- .../hdds/scm/TestStorageContainerManager.java | 24 +- .../apache/hadoop/hdds/scm/TestWatchForCommit.java | 54 +- .../hadoop/hdds/scm/TestXceiverClientGrpc.java | 75 +- .../hadoop/hdds/scm/TestXceiverClientManager.java | 4 +- .../hdds/scm/TestXceiverClientManagerSC.java | 4 +- .../hadoop/hdds/scm/TestXceiverClientMetrics.java | 118 +- .../apache/hadoop/hdds/scm/node/TestQueryNode.java | 23 + .../hadoop/hdds/scm/storage/TestCommitWatcher.java | 19 +- .../hdds/scm/storage/TestContainerCommandsEC.java | 2 +- .../org/apache/hadoop/ozone/TestCpuMetrics.java | 5 +- .../apache/hadoop/ozone/TestDelegationToken.java | 122 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 139 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 4 + .../hadoop/ozone/TestSecureOzoneCluster.java | 116 +- .../client/rpc/TestBlockDataStreamOutput.java | 42 +- ...stClientRetryContainerStateMachineFailures.java | 5 +- .../client/rpc/TestContainerStateMachine.java | 5 +- .../rpc/TestContainerStateMachineFailures.java | 5 +- .../rpc/TestContainerStateMachineStream.java | 10 +- .../ozone/client/rpc/TestECKeyOutputStream.java | 10 +- .../client/rpc/TestFailureHandlingByClient.java | 5 +- .../client/rpc/TestOzoneAtRestEncryption.java | 127 +- .../rpc/TestOzoneClientRetriesOnExceptions.java | 2 +- .../client/rpc/TestValidateBCSIDOnRestart.java | 5 +- .../ozone/client/rpc/read/InputStreamTests.java | 43 +- .../client/rpc/read/TestChunkInputStream.java | 71 +- .../client/rpc/read/TestLocalChunkInputStream.java | 100 +- .../rpc/read/TestStreamBlockInputStream.java | 145 +- .../ozone/client/rpc/read/TestStreamRead.java | 179 +- .../commandhandler/TestBlockDeletion.java | 4 +- .../TestContainerCommandReconciliation.java | 4 +- ...tBackgroundContainerDataScannerIntegration.java | 12 +- ...kgroundContainerMetadataScannerIntegration.java | 18 +- .../TestOnDemandContainerScannerIntegration.java | 32 +- .../freon/TestHadoopFsReadWriteValidator.java | 229 ++ .../ozone/om/OzoneManagerHAFollowerReadTests.java | 8 +- .../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 2 +- .../ozone/om/TestGetDBUpdatesAuthorization.java | 73 +- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 2 +- .../org/apache/hadoop/ozone/om/TestOmAcls.java | 4 +- .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 53 +- ...stOzoneManagerHAFollowerReadWithAllRunning.java | 45 +- ...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +- .../ozone/om/TestOzoneManagerHAWithAllRunning.java | 4 +- .../om/TestOzoneManagerHAWithStoppedNodes.java | 7 +- .../ozone/om/TestOzoneManagerRestInterface.java | 8 +- .../TestDirectoryDeletingServiceWithFSO.java | 97 +- .../TestOmMixedWorkloadUnderDeletionBench.java | 715 +++++ ...TestSnapshotDeletingServiceIntegrationTest.java | 131 +- .../hadoop/ozone/om/snapshot/OmSnapshotTests.java | 2 + .../snapshot/TestOmSnapshotDefragSpaceSavings.java | 713 +++++ .../om/snapshot/TestOzoneManagerHASnapshot.java | 32 +- .../TestSnapshotDirectoryCleaningService.java | 15 +- .../reconfig/TestDatanodeReconfiguration.java | 2 - .../hadoop/ozone/shell/TestOzoneRepairShell.java | 3 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 56 +- .../ozone/shell/TestTransferLeadershipShell.java | 60 +- .../tools/contract/AbstractContractDistCpTest.java | 2 +- .../org/apache/ozone/test/ClusterForTests.java | 5 +- .../java/org/apache/ozone/test/FreonTests.java | 9 + .../java/org/apache/ozone/test/KerberosTests.java | 191 ++ .../src/test/resources/ozone-site.xml | 90 +- hadoop-ozone/interface-client/pom.xml | 1 + .../src/main/proto/OmClientProtocol.proto | 153 + .../apache/hadoop/ozone/om/OMMetadataManager.java | 11 +- .../ozone/om/helpers/TestTransactionInfoCodec.java | 11 + hadoop-ozone/mini-cluster/pom.xml | 9 + .../hadoop/ozone/ClientConfigForTesting.java | 54 +- .../org/apache/hadoop/ozone/FixedHostMapping.java | 153 - .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 34 + .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 43 +- hadoop-ozone/ozone-manager/pom.xml | 16 +- .../org/apache/hadoop/ozone/audit/OMAction.java | 4 + .../hadoop/ozone/om/GrpcOzoneManagerServer.java | 40 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 74 +- .../org/apache/hadoop/ozone/om/ListIterator.java | 7 +- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 38 +- .../hadoop/ozone/om/OMPerformanceMetrics.java | 403 +-- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 50 +- .../apache/hadoop/ozone/om/OmMetadataReader.java | 167 +- .../org/apache/hadoop/ozone/om/OmSnapshot.java | 35 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 604 +++- .../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 34 +- .../org/apache/hadoop/ozone/om/ResolvedBucket.java | 34 + .../apache/hadoop/ozone/om/S3SecretManager.java | 4 + .../java/org/apache/hadoop/ozone/om/ScmClient.java | 81 +- .../hadoop/ozone/om/SnapshotChainManager.java | 163 +- .../hadoop/ozone/om/TrashOzoneFileSystem.java | 16 +- .../hadoop/ozone/om/codec/OMDBDefinition.java | 24 +- .../hadoop/ozone/om/execution/OMExecutionFlow.java | 20 +- .../hadoop/ozone/om/helpers/OMAuditLogger.java | 2 + .../hadoop/ozone/om/lock/OMLockDetailsUtil.java | 48 + .../hadoop/ozone/om/lock/OzoneManagerLock.java | 7 +- .../hadoop/ozone/om/ratis/OMThreadContext.java | 198 ++ .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 87 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 30 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 14 + .../hadoop/ozone/om/request/OMClientRequest.java | 93 +- .../om/request/bucket/OMBucketSetOwnerRequest.java | 7 +- .../request/bucket/OMBucketSetPropertyRequest.java | 9 +- .../request/bucket/acl/OMBucketAddAclRequest.java | 11 +- .../bucket/acl/OMBucketRemoveAclRequest.java | 11 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 9 +- .../om/request/file/OMDirectoryCreateRequest.java | 7 +- .../file/OMDirectoryCreateRequestWithFSO.java | 12 +- .../ozone/om/request/file/OMFileCreateRequest.java | 7 +- .../request/file/OMFileCreateRequestWithFSO.java | 5 +- .../om/request/file/OMRecoverLeaseRequest.java | 2 +- .../key/OMDirectoriesPurgeRequestWithFSO.java | 557 +++- .../ozone/om/request/key/OMKeyCommitRequest.java | 12 +- .../om/request/key/OMKeyCommitRequestWithFSO.java | 10 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 39 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 8 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 7 +- .../om/request/key/OMKeyDeleteRequestWithFSO.java | 5 +- .../ozone/om/request/key/OMKeyPurgeRequest.java | 8 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 29 +- .../ozone/om/request/key/OMKeySetTimesRequest.java | 2 +- .../ozone/om/request/key/OMKeysDeleteRequest.java | 15 +- .../om/request/key/acl/OMKeyAddAclRequest.java | 11 +- .../om/request/key/acl/OMKeyRemoveAclRequest.java | 11 +- .../om/request/key/acl/OMKeySetAclRequest.java | 11 +- .../OMLifecycleConfigurationDeleteRequest.java | 2 +- .../OMLifecycleConfigurationSetRequest.java | 2 +- .../lifecycle/OMLifecycleSaveScanStateRequest.java | 2 +- .../OMLifecycleSetServiceStatusRequest.java | 2 +- .../S3ExpiredMultipartUploadsAbortRequest.java | 333 +-- .../S3InitiateMultipartUploadRequestWithFSO.java | 5 +- .../multipart/S3MultipartUploadAbortRequest.java | 7 +- .../S3MultipartUploadCommitPartRequest.java | 7 +- .../S3MultipartUploadCompleteRequest.java | 26 +- .../om/request/s3/security/OMSetSecretRequest.java | 8 +- .../request/s3/security/S3AssumeRoleRequest.java | 611 ++++ .../security/S3DeleteRevokedSTSTokensRequest.java} | 55 +- .../om/request/s3/security/S3GetSecretRequest.java | 23 +- .../s3/security/S3RevokeSTSTokenRequest.java | 157 ++ .../request/s3/security/S3RevokeSecretRequest.java | 16 +- .../request/s3/security/S3SecretRequestHelper.java | 8 +- .../s3/tagging/S3BucketTaggingRequestBase.java | 2 +- .../security/OMCancelDelegationTokenRequest.java | 2 +- .../security/OMGetDelegationTokenRequest.java | 2 +- .../security/OMRenewDelegationTokenRequest.java | 2 +- .../snapshot/OMSnapshotMoveTableKeysRequest.java | 7 +- .../om/request/validation/ValidationCondition.java | 21 +- .../om/request/validation/VersionExtractor.java | 45 +- .../om/request/volume/OMQuotaRepairRequest.java | 11 + .../om/request/volume/OMVolumeDeleteRequest.java | 2 +- .../om/request/volume/OMVolumeSetOwnerRequest.java | 10 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 10 +- .../request/volume/acl/OMVolumeAddAclRequest.java | 6 +- .../volume/acl/OMVolumeRemoveAclRequest.java | 8 +- .../request/volume/acl/OMVolumeSetAclRequest.java | 6 +- .../hadoop/ozone/om/response/CleanupTableInfo.java | 51 - .../ozone/om/response/DummyOMClientResponse.java | 1 - .../hadoop/ozone/om/response/OMClientResponse.java | 24 +- .../om/response/bucket/OMBucketCreateResponse.java | 5 - .../om/response/bucket/OMBucketDeleteResponse.java | 6 - .../response/bucket/OMBucketSetOwnerResponse.java | 4 - .../bucket/OMBucketSetPropertyResponse.java | 4 - .../response/bucket/acl/OMBucketAclResponse.java | 4 - .../response/file/OMDirectoryCreateResponse.java | 4 - .../file/OMDirectoryCreateResponseWithFSO.java | 14 +- .../om/response/file/OMFileCreateResponse.java | 5 - .../response/file/OMFileCreateResponseWithFSO.java | 7 - .../om/response/file/OMRecoverLeaseResponse.java | 5 - .../om/response/key/OMAllocateBlockResponse.java | 5 - .../key/OMAllocateBlockResponseWithFSO.java | 5 - .../key/OMDirectoriesPurgeResponseWithFSO.java | 8 - .../ozone/om/response/key/OMKeyCommitResponse.java | 8 - .../response/key/OMKeyCommitResponseWithFSO.java | 8 - .../ozone/om/response/key/OMKeyCreateResponse.java | 6 - .../response/key/OMKeyCreateResponseWithFSO.java | 7 - .../ozone/om/response/key/OMKeyDeleteResponse.java | 7 - .../response/key/OMKeyDeleteResponseWithFSO.java | 10 - .../ozone/om/response/key/OMKeyPurgeResponse.java | 4 - .../ozone/om/response/key/OMKeyRenameResponse.java | 5 - .../response/key/OMKeyRenameResponseWithFSO.java | 7 - .../om/response/key/OMKeySetTimesResponse.java | 4 - .../response/key/OMKeySetTimesResponseWithFSO.java | 5 - .../om/response/key/OMKeysDeleteResponse.java | 7 - .../response/key/OMKeysDeleteResponseWithFSO.java | 11 - .../om/response/key/OMKeysRenameResponse.java | 4 - .../om/response/key/OMOpenKeysDeleteResponse.java | 8 - .../om/response/key/acl/OMKeyAclResponse.java | 4 - .../response/key/acl/OMKeyAclResponseWithFSO.java | 5 - .../key/acl/prefix/OMPrefixAclResponse.java | 4 - .../OMLifecycleConfigurationDeleteResponse.java | 4 - .../OMLifecycleConfigurationSetResponse.java | 4 - .../OMLifecycleSaveScanStateResponse.java | 3 - .../OMLifecycleSetServiceStatusResponse.java | 2 - .../S3ExpiredMultipartUploadsAbortResponse.java | 10 - .../S3InitiateMultipartUploadResponse.java | 5 - .../S3InitiateMultipartUploadResponseWithFSO.java | 7 - .../multipart/S3MultipartUploadAbortResponse.java | 9 - .../S3MultipartUploadAbortResponseWithFSO.java | 9 - .../S3MultipartUploadCommitPartResponse.java | 8 - ...S3MultipartUploadCommitPartResponseWithFSO.java | 9 - .../S3MultipartUploadCompleteResponse.java | 10 - .../S3MultipartUploadCompleteResponseWithFSO.java | 10 - .../response/s3/security/OMSetSecretResponse.java | 4 - .../security/S3AssumeRoleResponse.java} | 18 +- .../S3DeleteRevokedSTSTokensResponse.java} | 38 +- .../response/s3/security/S3GetSecretResponse.java | 4 - .../security/S3RevokeSTSTokenResponse.java} | 30 +- .../s3/security/S3RevokeSecretResponse.java | 4 - .../s3/tagging/S3DeleteObjectTaggingResponse.java | 4 - .../S3DeleteObjectTaggingResponseWithFSO.java | 4 - .../s3/tagging/S3PutObjectTaggingResponse.java | 4 - .../tagging/S3PutObjectTaggingResponseWithFSO.java | 4 - .../tenant/OMSetRangerServiceVersionResponse.java | 4 - .../s3/tenant/OMTenantAssignAdminResponse.java | 6 - .../tenant/OMTenantAssignUserAccessIdResponse.java | 10 - .../response/s3/tenant/OMTenantCreateResponse.java | 8 - .../response/s3/tenant/OMTenantDeleteResponse.java | 8 - .../s3/tenant/OMTenantRevokeAdminResponse.java | 6 - .../tenant/OMTenantRevokeUserAccessIdResponse.java | 10 - .../security/OMCancelDelegationTokenResponse.java | 4 - .../security/OMGetDelegationTokenResponse.java | 4 - .../security/OMRenewDelegationTokenResponse.java | 4 - .../snapshot/OMSnapshotCreateResponse.java | 7 - .../snapshot/OMSnapshotDeleteResponse.java | 4 - .../OMSnapshotMoveDeletedKeysResponse.java | 3 - .../snapshot/OMSnapshotMoveTableKeysResponse.java | 3 - .../response/snapshot/OMSnapshotPurgeResponse.java | 8 - .../snapshot/OMSnapshotRenameResponse.java | 4 - .../snapshot/OMSnapshotSetPropertyResponse.java | 4 - .../response/upgrade/OMCancelPrepareResponse.java | 4 - .../upgrade/OMFinalizeUpgradeResponse.java | 3 - .../om/response/upgrade/OMPrepareResponse.java | 3 - .../om/response/util/OMEchoRPCWriteResponse.java | 2 - .../om/response/volume/OMQuotaRepairResponse.java | 5 - .../om/response/volume/OMVolumeAclOpResponse.java | 4 - .../om/response/volume/OMVolumeCreateResponse.java | 4 - .../om/response/volume/OMVolumeDeleteResponse.java | 4 - .../response/volume/OMVolumeSetOwnerResponse.java | 4 - .../response/volume/OMVolumeSetQuotaResponse.java | 4 - .../ozone/om/service/DirectoryDeletingService.java | 58 +- .../ozone/om/service/KeyLifecycleService.java | 12 +- .../om/service/KeyLifecycleServiceMetrics.java | 31 +- .../hadoop/ozone/om/service/QuotaRepairTask.java | 256 +- .../om/service/RevokedSTSTokenCleanupService.java | 269 ++ .../hadoop/ozone/om/snapshot/ReferenceCounted.java | 4 +- .../ozone/om/snapshot/SnapshotDiffManager.java | 22 +- .../hadoop/ozone/om/snapshot/diff/EntryValue.java | 150 + .../om/snapshot/diff/FullDiffSequentialReader.java | 205 ++ .../ozone/om/snapshot/diff/SnapDiffJobStore.java | 370 +++ .../{ => diff}/SnapshotDiffValueParser.java | 35 +- .../trapped/BucketDeletedDataCalculator.java | 373 +++ .../ozone/om/snapshot/trapped/package-info.java | 8 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 47 +- .../protocolPB/OzoneManagerRequestHandler.java | 89 +- .../hadoop/ozone/security/S3SecurityUtil.java | 124 + .../hadoop/ozone/security/STSSecurityUtil.java | 344 +++ .../hadoop/ozone/security/STSTokenEncryption.java | 205 ++ .../hadoop/ozone/security/STSTokenIdentifier.java | 482 ++++ .../ozone/security/STSTokenSecretManager.java | 234 ++ .../apache/hadoop/ozone/om/FaultInjectorImpl.java | 12 + .../hadoop/ozone/om/TestEDEKCacheLoader.java | 78 + .../ozone/om/TestGrpcOzoneManagerServer.java | 152 + .../hadoop/ozone/om/TestOMMetadataReader.java | 724 ++++- ...anagerServer.java => TestOMMetricsSetters.java} | 48 +- .../hadoop/ozone/om/TestOMPerformanceMetrics.java | 107 + .../hadoop/ozone/om/TestOmMetadataManager.java | 43 + .../ozone/om/TestOmSnapshotLeakDetection.java | 87 + .../hadoop/ozone/om/TestOzoneManagerS3Auth.java | 123 + .../ozone/om/TestOzoneManagerSnapshotCount.java | 149 + .../org/apache/hadoop/ozone/om/TestScmClient.java | 24 + .../om/TestTrashOzoneFileSystemLegacyBucket.java | 276 ++ .../ozone/om/execution/TestOMExecutionFlow.java | 165 ++ .../ozone/om/lock/TestOMLockDetailsUtil.java | 78 + .../hadoop/ozone/om/ratis/TestOMThreadContext.java | 207 ++ ...estOzoneManagerDoubleBufferTransactionInfo.java | 235 ++ ...tOzoneManagerDoubleBufferWithDummyResponse.java | 74 +- .../om/ratis/TestOzoneManagerRatisRequest.java | 58 + .../om/ratis/TestOzoneManagerRatisServer.java | 50 + .../om/ratis/TestOzoneManagerStateMachine.java | 493 +++- .../apache/hadoop/ozone/om/ratis/package-info.java | 8 +- .../request/TestOMClientRequestWithUserInfo.java | 335 ++- .../file/TestOMDirectoryCreateRequestWithFSO.java | 28 + .../key/TestOMDirectoriesPurgeApplyPerf.java | 226 ++ .../TestOMDirectoriesPurgeRequestAndResponse.java | 86 + .../om/request/key/TestOMKeyCommitRequest.java | 34 + .../om/request/key/TestOMKeyCreateRequest.java | 102 +- .../om/request/key/TestOMKeyDeleteRequest.java | 8 + .../key/TestOMKeyPurgeRequestAndResponse.java | 67 +- .../om/request/key/TestOMKeysDeleteRequest.java | 48 + .../key/TestOMKeysDeleteRequestWithFSO.java | 16 + .../request/key/TestOMOpenKeysDeleteRequest.java | 7 + ...> OMLifecycleConfigurationRequestTestBase.java} | 2 +- .../TestOMLifecycleConfigurationDeleteRequest.java | 2 +- .../TestOMLifecycleConfigurationSetRequest.java | 2 +- .../TestS3ExpiredMultipartUploadsAbortRequest.java | 141 +- ...estS3InitiateMultipartUploadRequestWithFSO.java | 5 + .../TestS3MultipartUploadAbortRequest.java | 10 + .../TestS3MultipartUploadCompleteRequest.java | 149 +- ...estS3MultipartUploadCompleteRequestWithFSO.java | 48 + .../request/s3/security/S3SecurityTestUtils.java} | 37 +- .../s3/security/TestS3AssumeRoleRequest.java | 988 +++++++ .../s3/security/TestS3GetSecretRequest.java | 34 + .../s3/security/TestS3RevokeSTSTokenRequest.java | 402 +++ .../s3/security/TestS3SecretRequestHelper.java | 31 +- .../ozone/om/request/s3/security/package-info.java | 8 +- .../request/volume/TestOMQuotaRepairRequest.java | 55 + .../ozone/om/response/TestCleanupTableInfo.java | 380 --- .../ozone/om/response/TestOMClientResponse.java} | 41 +- .../hadoop/ozone/om/response/TestOMResponse.java | 51 +- .../hadoop/ozone/om/response/TestS3DerivedKey.java | 241 ++ .../file/TestOMDirectoryCreateResponseWithFSO.java | 2 - .../s3/security/TestS3AssumeRoleResponse.java | 133 + .../om/service/TestDirectoryDeletingService.java | 60 +- .../ozone/om/service/TestKeyDeletingService.java | 39 + .../ozone/om/service/TestKeyLifecycleService.java | 414 ++- .../ozone/om/service/TestQuotaRepairTask.java | 538 ++++ .../service/TestRevokedSTSTokenCleanupService.java | 449 +++ .../hadoop/ozone/om/service/package-info.java | 8 +- .../snapshot/TestOmSnapshotLocalDataManager.java | 3 +- .../ozone/om/snapshot/TestSnapshotChain.java | 45 + .../ozone/om/snapshot/TestSnapshotDiffManager.java | 18 + .../diff/TestFullDiffSequentialReader.java | 320 +++ .../{ => diff}/TestSnapshotDiffValueParser.java | 2 +- .../trapped/TestBucketDeletedDataCalculator.java | 243 ++ .../protocolPB/TestOzoneManagerRequestHandler.java | 4 +- .../hadoop/ozone/security/TestS3SecurityUtil.java | 405 +++ .../hadoop/ozone/security/TestSTSSecurityUtil.java | 608 ++++ .../ozone/security/TestSTSTokenEncryption.java | 215 ++ .../ozone/security/TestSTSTokenIdentifier.java | 633 +++++ .../ozone/security/TestSTSTokenSecretManager.java | 213 ++ .../hadoop/ozone/security/acl/package-info.java | 8 +- hadoop-ozone/ozonefs-common/pom.xml | 7 + ...sDtFetcher.java => AbstractOzoneDtFetcher.java} | 41 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 8 + .../ozone/BasicRootedOzoneClientAdapterImpl.java | 44 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 15 +- .../org/apache/hadoop/fs/ozone/O3DtFetcher.java | 48 + .../org/apache/hadoop/fs/ozone/O3fsDtFetcher.java | 48 +- .../org/apache/hadoop/fs/ozone/OfsDtFetcher.java | 14 +- .../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 179 +- .../hadoop/fs/ozone/TestBasicOzoneFileSystems.java | 105 + .../TestBasicRootedOzoneClientAdapterHeadOp.java | 49 +- .../apache/hadoop/fs/ozone/TestOzoneDtFetcher.java | 140 + .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 299 ++ hadoop-ozone/ozonefs-hadoop2/pom.xml | 3 + hadoop-ozone/ozonefs-hadoop3/pom.xml | 1 + .../org.apache.hadoop.security.token.DtFetcher | 2 + hadoop-ozone/ozonefs-shaded/pom.xml | 1 + .../org.apache.hadoop.security.token.DtFetcher | 2 + .../hadoop/fs/ozone/TestDtFetcherProviders.java} | 34 +- hadoop-ozone/pom.xml | 18 +- hadoop-ozone/recon-codegen/pom.xml | 14 +- hadoop-ozone/recon/pom.xml | 12 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 2 +- .../hadoop/ozone/recon/api/BucketEndpoint.java | 1 - .../ozone/recon/api/ClusterStateEndpoint.java | 2 +- .../ozone/recon/api/UtilizationEndpoint.java | 12 +- .../hadoop/ozone/recon/api/VolumeEndpoint.java | 1 - .../ozone/recon/api/filters/ReconAuthFilter.java | 2 +- .../hadoop/ozone/recon/api/types/EntityType.java | 87 +- .../ozone/recon/api/types/FeatureProvider.java | 10 +- .../ozone/recon/fsck/ReconReplicationManager.java | 36 +- .../ozone/recon/scm/ReconContainerManager.java | 8 +- .../impl/ReconContainerMetadataManagerImpl.java | 7 - .../recon/tasks/MultipartInfoInsightHandler.java | 118 +- .../ozone/recon/tasks/OmTableInsightTask.java | 32 +- .../recon/upgrade/ReconLayoutVersionManager.java | 95 +- .../upgrade/ReconTaskStatusTableUpgradeAction.java | 87 +- .../webapps/recon/ozone-recon-web/package.json | 4 +- .../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 34 +- .../src/v2/components/navBar/navBar.tsx | 12 +- .../src/v2/pages/heatmap/heatmap.tsx | 35 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 49 +- .../ozone/recon/api/TestOmDBInsightEndPoint.java | 40 +- .../recon/api/types/TestFeatureProvider.java} | 32 +- .../recon/fsck/TestReconReplicationManager.java | 50 + .../persistence/TestStatsSchemaDefinition.java | 22 +- .../TestUtilizationSchemaDefinition.java | 18 +- .../ozone/recon/scm/TestReconContainerManager.java | 7 +- .../TestStorageContainerServiceProviderImpl.java | 53 + .../recon/tasks/TestContainerKeyMapperTask.java | 21 +- .../recon/tasks/TestContainerSizeCountTask.java | 2 +- .../ozone/recon/tasks/TestOmTableInsightTask.java | 118 +- .../upgrade/TestReconLayoutVersionManager.java | 23 +- .../TestReconTaskStatusTableUpgradeAction.java | 176 ++ hadoop-ozone/s3gateway/pom.xml | 30 +- .../org/apache/hadoop/ozone/audit/S3GAction.java | 11 +- .../hadoop/ozone/s3/EmptyContentTypeFilter.java | 30 +- .../java/org/apache/hadoop/ozone/s3/Gateway.java | 5 +- .../hadoop/ozone/s3/MultiDigestInputStream.java | 8 + .../hadoop/ozone/s3/OzoneClientProducer.java | 3 +- .../hadoop/ozone/s3/ReadConsistencyFilter.java | 53 + .../hadoop/ozone/s3/S3ContentTypeFilter.java | 135 + .../hadoop/ozone/s3/S3GatewayHttpServer.java | 13 + .../hadoop/ozone/s3/S3GatewayReadinessProbe.java | 164 ++ .../hadoop/ozone/s3/S3GatewayWebAdminServer.java | 90 +- ...Mapper.java => S3STSHeadersResponseFilter.java} | 37 +- .../apache/hadoop/ozone/s3/S3STSHttpServer.java | 108 + .../hadoop/ozone/s3/SignedChunksInputStream.java | 403 ++- .../hadoop/ozone/s3/VirtualHostStyleFilter.java | 46 +- .../apache/hadoop/ozone/s3/WeldCdiIntegration.java | 50 + .../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 5 + .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 167 +- .../s3/endpoint/BucketGetLocationHandler.java | 5 + .../ozone/s3/endpoint/BucketLifecycleHandler.java | 14 +- .../ozone/s3/endpoint/BucketOperationHandler.java | 21 +- .../BucketOperationNotImplementedHandler.java | 64 + .../hadoop/ozone/s3/endpoint/CopyPartResult.java | 4 +- .../hadoop/ozone/s3/endpoint/CopyResult.java} | 25 +- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 217 +- .../s3/endpoint/GetObjectAttributesResponse.java | 10 +- .../s3/endpoint/ListMultipartUploadsHandler.java | 15 +- .../ozone/s3/endpoint/MultipartKeyHandler.java | 14 +- .../hadoop/ozone/s3/endpoint/ObjectAclHandler.java | 15 +- .../ozone/s3/endpoint/ObjectAttributesHandler.java | 187 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 264 +- .../ozone/s3/endpoint/ObjectEndpointStreaming.java | 72 +- .../ozone/s3/endpoint/ObjectGetTorrentHandler.java | 5 + .../ozone/s3/endpoint/ObjectOperationHandler.java | 17 +- .../ObjectOperationNotImplementedHandler.java | 65 + .../ozone/s3/endpoint/ObjectTaggingHandler.java | 19 +- .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 23 +- .../s3/endpoint/S3LifecycleConfiguration.java | 33 +- .../s3/endpoint/S3ObjectStreamingWriteGuard.java | 5 + .../ozone/s3/endpoint/S3ObjectWriteGuard.java | 30 + .../hadoop/ozone/s3/endpoint/S3RequestContext.java | 14 + .../hadoop/ozone/s3/exception/OS3Exception.java | 50 +- .../ozone/s3/exception/OS3ExceptionMapper.java | 18 +- .../hadoop/ozone/s3/exception/OSTSException.java | 173 ++ ...ceptionMapper.java => OSTSExceptionMapper.java} | 11 +- .../s3/exception/OSTSNotFoundExceptionMapper.java | 93 + .../hadoop/ozone/s3/exception/S3ErrorTable.java | 60 +- .../ozone/s3/signature/AWSSignatureProcessor.java | 118 +- .../s3/signature/AuthorizationV4HeaderParser.java | 44 +- .../s3/signature/AuthorizationV4QueryParser.java | 9 +- .../hadoop/ozone/s3/signature/ChunksValidator.java | 151 + .../hadoop/ozone/s3/signature/Credential.java | 20 + .../s3/signature/MalformedResourceException.java | 19 +- .../hadoop/ozone/s3/signature/SignatureInfo.java | 61 +- .../ozone/s3/signature/SignatureProcessor.java | 4 +- .../ozone/s3/signature/StringToSignProducer.java | 101 +- .../ozone/s3/util/RangeHeaderParserUtil.java | 6 +- .../ozone/s3/util/ReadConsistencyContext.java | 112 + .../org/apache/hadoop/ozone/s3/util/S3Consts.java | 54 +- .../hadoop/ozone/s3/util/S3GActionIamMapper.java | 95 + .../org/apache/hadoop/ozone/s3/util/S3Utils.java | 2 + .../ozone/s3secret/S3SecretEndpointBase.java | 32 + .../ozone/s3secret/S3SecretManagementEndpoint.java | 17 +- .../apache/hadoop/ozone/s3sts/Application.java} | 29 +- .../ozone/s3sts/S3AssumeRoleResponseXml.java | 162 ++ .../s3sts/S3GetCallerIdentityResponseXml.java | 92 + .../apache/hadoop/ozone/s3sts/S3STSConfigKeys.java | 54 + .../apache/hadoop/ozone/s3sts/S3STSEnabled.java | 18 +- .../s3sts/S3STSEnabledEndpointRequestFilter.java | 53 + .../apache/hadoop/ozone/s3sts/S3STSEndpoint.java | 521 ++++ .../S3STSEndpointBase.java} | 72 +- .../hadoop/ozone/s3sts/S3STSResponseMetadata.java} | 32 +- .../apache/hadoop/ozone/s3sts/package-info.java | 19 +- .../webapps/{s3gateway => s3g-sts}/WEB-INF/web.xml | 17 +- .../resources/webapps/s3gateway/WEB-INF/web.xml | 8 + .../hadoop/ozone/client/ClientProtocolStub.java | 133 +- .../hadoop/ozone/client/OzoneBucketStub.java | 227 +- .../ozone/client/OzoneDataStreamOutputStub.java | 18 +- .../hadoop/ozone/client/OzoneOutputStreamStub.java | 20 +- .../hadoop/ozone/s3/TestAuthorizationFilter.java | 120 +- .../ozone/s3/TestEmptyContentTypeFilter.java | 35 + .../hadoop/ozone/s3/TestOzoneClientProducer.java | 58 + .../hadoop/ozone/s3/TestReadConsistencyFilter.java | 51 + .../hadoop/ozone/s3/TestS3ContentTypeFilter.java | 127 + .../hadoop/ozone/s3/TestS3GatewayHttpServers.java | 100 + .../ozone/s3/TestS3GatewayLivenessServlet.java | 126 + .../ozone/s3/TestSignedChunksInputStream.java | 354 ++- .../ozone/s3/TestVirtualHostStyleFilter.java | 61 +- .../ozone/s3/endpoint/EndpointTestUtils.java | 40 + .../hadoop/ozone/s3/endpoint/TestBucketList.java | 389 ++- .../s3/endpoint/TestBucketNotImplemented.java | 102 + .../ozone/s3/endpoint/TestCopyActionsAudit.java | 133 + ...estCopyObjectAndUploadPartCopyLastModified.java | 150 + .../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 264 ++ .../s3/endpoint/TestListMultipartUploads.java | 166 ++ .../hadoop/ozone/s3/endpoint/TestListParts.java | 108 + .../s3/endpoint/TestMultipartUploadWithCopy.java | 6 +- .../ozone/s3/endpoint/TestObjectAttributesGet.java | 273 ++ .../ozone/s3/endpoint/TestObjectMultiDelete.java | 108 + .../s3/endpoint/TestObjectNotImplemented.java | 113 + .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 395 ++- .../ozone/s3/endpoint/TestObjectTaggingDelete.java | 39 +- .../ozone/s3/endpoint/TestObjectTaggingGet.java | 20 + .../ozone/s3/endpoint/TestObjectTaggingPut.java | 35 +- .../hadoop/ozone/s3/endpoint/TestPartUpload.java | 108 +- .../ozone/s3/endpoint/TestPermissionCheck.java | 82 +- .../hadoop/ozone/s3/endpoint/TestRootList.java | 111 +- .../TestS3ActionOverrideForOwnerVerification.java | 174 ++ .../TestS3LifecycleConfigurationDelete.java | 125 + .../endpoint/TestS3LifecycleConfigurationGet.java | 90 + .../endpoint/TestS3LifecycleConfigurationPut.java | 184 +- .../ozone/s3/endpoint/TestS3ObjectWriteGuard.java | 31 + .../ozone/s3/endpoint/TestS3RequestContext.java | 86 + .../ozone/s3/endpoint/TestUploadWithStream.java | 137 +- .../ozone/s3/exception/TestOS3Exceptions.java | 61 +- .../ozone/s3/exception/TestOSTSExceptions.java | 104 + .../exception/TestOSTSNotFoundExceptionMapper.java | 120 + .../ozone/s3/signature/SignatureTestUtils.java | 162 ++ .../s3/signature/TestAWSSignatureProcessor.java | 14 + .../signature/TestAuthorizationV4HeaderParser.java | 133 +- .../signature/TestAuthorizationV4QueryParser.java | 55 +- .../ozone/s3/signature/TestChunksValidator.java | 194 ++ .../s3/signature/TestStringToSignProducer.java | 114 +- .../ozone/s3/util/TestRangeHeaderParserUtil.java | 7 + .../ozone/s3/util/TestS3GActionIamMapper.java | 71 + .../apache/hadoop/ozone/s3/util/TestS3Utils.java | 9 + .../apache/hadoop/ozone/s3/util/package-info.java | 8 +- .../ozone/s3secret/TestS3SecretEndpointBase.java | 69 + .../hadoop/ozone/s3secret/TestSecretGenerate.java | 6 +- .../hadoop/ozone/s3secret/TestSecretRevoke.java | 5 + .../hadoop/ozone/s3sts/TestS3STSEndpoint.java | 994 +++++++ .../apache/hadoop/ozone/s3sts/package-info.java | 8 +- hadoop-ozone/tools/pom.xml | 5 + .../hadoop/ozone/local/LocalOzoneCluster.java | 303 +- .../org/apache/hadoop/ozone/local/OzoneLocal.java | 76 +- .../hadoop/ozone/local/TestLocalOzoneCluster.java | 301 +- .../apache/hadoop/ozone/local/TestOzoneLocal.java | 126 +- pom.xml | 179 +- 966 files changed, 62588 insertions(+), 8251 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
