This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a change to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 79f60c4 HDDS-6271. EC: Apply fix for HDFS-16422 to the Ozone EC
libraries (#3087)
add 634d666 HDDS-6239. ozonesecure-mr failing with No URLs in mirrorlist
(#3029)
add 5d496f2 HDDS-6201. Fix NPE for DataScanner with scanned container
deleted by others. (#3005)
add 9186e6b HDDS-5529. For any IOexception from @Replicated method we
should throw it (#2788)
add 1804e22 HDDS-6181. Change SCMHAInvocationHandler#invokeRatis()
logging to TRACE (#2992)
add e47b6f0 HDDS-6206. Application errors must not flood system log
(#3001)
add f757d99 HDDS-6245. Add BucketLayout logging to Audit Logs (#3040)
add aafc21a HDDS-6238 Reduce memory requirements for list keys. (#3032)
add 7cee0ea HDDS-2919. Intermittent failure in TestRDBStore (#3028)
add 5a35cab HDDS-6253. Unnecessary duplicate smoketest after defaulting
to FSO (#3036)
add 5b1dbeb HDDS-6204. Cleanup handling malformed authorization header
(#2999)
add b32e135 HDDS-6169. Selective checks: skip junit tests on ozone-runner
image update (#2974)
add 14fa87c HDDS-6270. Use a dedicated file instead of /etc/passwd for
xcompat acceptance test (#3050)
add 5c3aa01 HDDS-6273. Amend doc SecuringTDE.md (#3047)
add c5fb612 HDDS-6140. Selective checks: skip unit check for
integration-test changes (#2948)
add 11f4c9b HDDS-6215. Recon get limited delta updates from OM (#3009)
add 40d0a40 HDDS-6226. Run tests for selective CI checks in CI (#3019)
add 86a771d HDDS-6247. Avoid logging stack trace for user input problems
(#3039)
add de42c61 HDDS-6208. New checkstyle: WhitespaceAround (#3003)
add ad6d3bc HDDS-6289. Upgrade acceptance test log flooded with parse
error (#3063)
add 5f8e637 HDDS-6291. Do not let slf4j-log4j12 leak into Ozone FS shaded
jar (#3064)
add 59cae07 HDDS-6285. ozonesecure-mr intermittently failing with timeout
downloading packages (#3057)
add a5accd2 HDDS-6278 Improve memory profile for listStatus API call.
(#3053)
add 594c4a9 HDDS-6257. Wrong stack trace for S3 errors (#3066)
add f31d589 HDDS-6088. Implement O3FS/OFS getFileChecksum() using file
checksum helpers. (#2935)
add 7c990ab HDDS-6088. Implement O3FS/OFS getFileChecksum() using file
checksum helpers - addendum: fix checkstyle
add f7e1fa0 HDDS-4126. Freon key generator should support >2GB files.
(#3054)
add 9ce981c HDDS-6293. Allow using custom ozone-runner image (#3072)
add fa2ae7b HDDS-6309. Update ozone-runner version to 20220212-1 (#3079)
add 3ed0b6b HDDS-6304. Add enforcer to make sure ozone.version equals
project.version (#3075)
add 49221c0 HDDS-6190. Cleanup unnecessary datanode id path checks.
(#2993)
add e7f5b4e HDDS-6310. Update json-smart to 2.4.7. (#3080)
add 917ce68 HDDS-6186. Selective checks: skip integration check for unit
test changes (#3061)
add 41651e8 HDDS-6313. Remove replicas in ContainerStateMap when a
container is deleted (#3086)
add c04c01d HDDS-5913. Avoid integer overflow when setting
dfs.container.ratis.lo… (#2785)
add e0d4007 HDDS-6322. Fix Recon getting inccorrect sequenceNumber from
OM (#3090)
new aebaff5 Merge master to EC branch HDDS-3816-ec
new b2a33b7 HDDS-6320: EC: Fix new checkstyle rule warnings in EC branch
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 6 ++
.gitignore | 3 +-
dev-support/ci/selective_ci_checks.bats | 70 ++++++++++++-
dev-support/ci/selective_ci_checks.sh | 76 +++++++++++----
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 38 ++++++++
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 8 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 10 +-
.../apache/hadoop/hdds/scm/storage/BufferPool.java | 2 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 2 +-
.../hdds/scm/storage/RatisBlockOutputStream.java | 2 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +-
.../java/org/apache/hadoop/hdds/StringUtils.java | 2 +-
.../hadoop/hdds/annotation/InterfaceAudience.java | 6 +-
.../hdds/client/DefaultReplicationConfig.java | 2 +-
.../org/apache/hadoop/hdds/client/OzoneQuota.java | 2 +-
.../org/apache/hadoop/hdds/client/QuotaList.java | 11 ++-
.../hadoop/hdds/client/ReplicationFactor.java | 2 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 2 +-
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 2 +-
.../hadoop/hdds/protocol/DatanodeDetails.java | 4 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 2 +-
.../org/apache/hadoop/hdds/recon/ReconConfig.java | 2 +-
.../hadoop/hdds/scm/ByteStringConversion.java | 2 +-
.../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 2 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +-
.../scm/container/ReplicationManagerReport.java | 2 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 14 +--
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 28 +++---
.../org/apache/hadoop/hdds/scm/net/NodeSchema.java | 4 +-
.../hadoop/hdds/scm/net/NodeSchemaLoader.java | 8 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 8 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 8 +-
.../apache/hadoop/hdds/utils/HddsVersionInfo.java | 2 +-
.../hadoop/hdds/utils/ResourceSemaphore.java | 10 +-
.../org/apache/hadoop/hdds/utils/UniqueId.java | 2 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 11 ++-
.../hadoop/ozone/audit/AuditEventStatus.java | 2 +-
.../org/apache/hadoop/ozone/audit/AuditLogger.java | 4 +-
.../apache/hadoop/ozone/audit/AuditLoggerType.java | 2 +-
.../org/apache/hadoop/ozone/audit/AuditMarker.java | 4 +-
.../apache/hadoop/ozone/audit/AuditMessage.java | 10 +-
.../hadoop/ozone/common/ChecksumByteBuffer.java | 2 +-
.../ozone/common/IncrementalChunkBuffer.java | 13 +--
.../apache/hadoop/ozone/common/StorageInfo.java | 8 +-
.../ozone/common/ha/ratis/RatisSnapshotInfo.java | 2 +-
.../ozone/container/common/helpers/BlockData.java | 4 +-
.../helpers/ContainerCommandRequestPBHelper.java | 10 +-
.../java/org/apache/hadoop/ozone/lease/Lease.java | 8 +-
.../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +-
.../hadoop/ozone/util/ShutdownHookManager.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 18 ++++
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 4 +-
.../hadoop/hdds/client/TestReplicationConfig.java | 4 +-
.../java/org/apache/hadoop/hdds/fs/TestDU.java | 2 +-
.../ratis/TestContainerCommandRequestMessage.java | 6 +-
.../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 4 +-
.../hdds/scm/net/TestNetworkTopologyImpl.java | 20 ++--
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 4 +-
.../hadoop/hdds/utils/MockGatheringChannel.java | 2 +-
.../hadoop/hdds/utils/TestResourceSemaphore.java | 6 +-
.../org/apache/hadoop/ozone/audit/DummyEntity.java | 2 +-
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 4 +-
.../apache/hadoop/ozone/common/TestChecksum.java | 2 +-
.../hadoop/ozone/common/TestChunkBuffer.java | 20 ++--
.../hadoop/ozone/common/TestStateMachine.java | 4 +-
.../ozone/container/ContainerTestHelper.java | 2 +-
.../hadoop/ozone/lease/TestLeaseManager.java | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 27 +-----
.../container/common/helpers/ContainerMetrics.java | 6 +-
.../container/common/helpers/ContainerUtils.java | 2 +-
.../container/common/impl/HddsDispatcher.java | 15 +--
.../common/impl/OpenContainerBlockMap.java | 2 +-
.../common/statemachine/DatanodeConfiguration.java | 2 +-
.../common/statemachine/DatanodeStateMachine.java | 8 +-
.../common/statemachine/StateContext.java | 12 +--
.../commandhandler/CommandDispatcher.java | 2 +-
.../SetNodeOperationalStateCommandHandler.java | 11 ---
.../common/states/datanode/InitDatanodeState.java | 9 +-
.../states/endpoint/VersionEndpointTask.java | 2 +-
.../common/transport/server/ratis/CSMMetrics.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 4 +-
.../transport/server/ratis/XceiverServerRatis.java | 9 +-
.../container/common/utils/HddsVolumeUtil.java | 2 +-
.../container/common/volume/MutableVolumeSet.java | 2 +-
.../container/common/volume/StorageVolume.java | 2 +-
.../ozone/container/common/volume/VolumeUsage.java | 2 +-
.../container/keyvalue/KeyValueContainer.java | 6 +-
.../container/keyvalue/KeyValueContainerCheck.java | 6 +-
.../container/keyvalue/KeyValueContainerData.java | 2 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 6 +-
.../helpers/KeyValueContainerLocationUtil.java | 2 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 8 +-
.../background/BlockDeletingService.java | 4 +-
.../container/metadata/AbstractDatanodeStore.java | 2 +-
.../container/ozoneimpl/ContainerController.java | 14 ++-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 6 +-
.../ozoneimpl/ContainerMetadataScanner.java | 2 +-
.../ContainerMetadataScrubberMetrics.java | 2 +-
.../replication/GrpcReplicationClient.java | 2 +-
.../container/stream/DirstreamClientHandler.java | 2 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 2 +-
.../upgrade/VersionedDatanodeFeatures.java | 2 +-
.../ozone/protocol/commands/ReregisterCommand.java | 2 +-
.../hadoop/ozone/TestHddsSecureDatanodeInit.java | 2 +-
.../hadoop/ozone/container/common/ScmTestMock.java | 35 ++++---
.../container/common/TestBlockDeletingService.java | 7 +-
.../ozone/container/common/TestContainerCache.java | 2 +-
.../container/common/TestDatanodeStateMachine.java | 5 -
.../TestSchemaOneBackwardsCompatibility.java | 36 ++++---
.../container/common/helpers/TestBlockData.java | 6 +-
.../common/helpers/TestDatanodeVersionFile.java | 10 +-
.../container/common/impl/TestContainerSet.java | 12 +--
.../container/common/impl/TestHddsDispatcher.java | 2 +-
.../container/common/interfaces/TestHandler.java | 2 +-
.../common/report/TestReportPublisher.java | 2 +-
.../TestCloseContainerCommandHandler.java | 4 +-
.../volume/TestRoundRobinVolumeChoosingPolicy.java | 2 +-
.../container/common/volume/TestVolumeSet.java | 2 +-
.../keyvalue/TestKeyValueBlockIterator.java | 14 +--
.../container/keyvalue/TestKeyValueContainer.java | 2 +-
.../keyvalue/TestKeyValueContainerCheck.java | 2 +-
.../container/keyvalue/TestKeyValueHandler.java | 4 +-
.../keyvalue/impl/CommonChunkManagerTestCases.java | 4 +-
.../container/ozoneimpl/TestContainerReader.java | 8 +-
.../replication/TestGrpcOutputStream.java | 8 +-
.../replication/TestReplicationSupervisor.java | 2 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 6 +-
hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 1 +
hadoop-hdds/docs/content/security/SecuringTDE.md | 48 ++++++---
.../docs/content/security/SecuringTDE.zh.md | 4 +-
.../erasurecode/rawcoder/util/GaloisField.java | 23 ++---
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 3 +-
.../SCMSecurityProtocolClientSideTranslatorPB.java | 2 +-
...inerLocationProtocolClientSideTranslatorPB.java | 4 +-
.../scm/update/client/CRLClientUpdateHandler.java | 2 +-
.../hdds/scm/update/client/ClientCRLStore.java | 4 +-
.../update/client/SCMUpdateServiceGrpcClient.java | 2 +-
.../x509/certificate/authority/BaseApprover.java | 2 +-
.../certificate/authority/DefaultCAServer.java | 6 +-
.../authority/PKIProfiles/DefaultCAProfile.java | 2 +-
.../client/DefaultCertificateClient.java | 32 +++---
.../certificate/client/OMCertificateClient.java | 4 +-
.../certificates/utils/CertificateSignRequest.java | 10 +-
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 2 +-
.../hdds/security/x509/crl/CRLInfoCodec.java | 2 +-
.../hadoop/hdds/server/http/ProfileServlet.java | 2 +-
.../hadoop/hdds/utils/DBCheckpointMetrics.java | 2 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 2 +-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 5 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +-
.../apache/hadoop/hdds/utils/TransactionInfo.java | 2 +-
.../hadoop/hdds/utils/db/DBConfigFromFile.java | 4 +-
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 4 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +-
.../hadoop/hdds/utils/db/cache/CacheKey.java | 2 +-
.../hadoop/hdds/utils/db/cache/EpochEntry.java | 2 +-
.../x509/certificate/authority/MockCAStore.java | 2 +-
.../x509/certificate/utils/TestCRLCodec.java | 2 +-
.../certificates/TestCertificateSignRequest.java | 4 +-
.../x509/certificates/TestRootCertificate.java | 2 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 2 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 10 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 40 +-------
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 6 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 4 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 2 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 20 ++--
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +-
.../hdds/scm/container/ContainerReplicaCount.java | 18 ++--
.../hdds/scm/container/ContainerReportHandler.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 24 ++---
.../balancer/AbstractFindTargetGreedy.java | 6 +-
.../scm/container/balancer/ContainerBalancer.java | 2 +-
.../balancer/ContainerBalancerConfiguration.java | 4 +-
.../scm/container/balancer/FindSourceGreedy.java | 6 +-
.../ContainerPlacementPolicyFactory.java | 6 +-
.../algorithms/SCMContainerPlacementMetrics.java | 2 +-
.../algorithms/SCMContainerPlacementRackAware.java | 12 +--
.../SCMContainerPlacementRackScatter.java | 8 +-
.../container/placement/metrics/SCMMetrics.java | 2 +-
.../scm/container/states/ContainerStateMap.java | 3 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 9 +-
.../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 2 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 4 +-
.../hdds/scm/metadata/X509CertificateCodec.java | 2 +-
.../apache/hadoop/hdds/scm/node/CommandQueue.java | 2 +-
.../hdds/scm/node/NodeDecommissionManager.java | 32 +++---
.../apache/hadoop/hdds/scm/node/NodeManager.java | 2 +-
.../hadoop/hdds/scm/node/NodeStateManager.java | 2 +-
.../apache/hadoop/hdds/scm/node/NodeStatus.java | 4 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 8 +-
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 8 +-
.../hdds/scm/node/SCMNodeStorageStatMap.java | 2 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 2 +-
.../hadoop/hdds/scm/node/states/NodeStateMap.java | 2 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 4 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 7 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +-
.../scm/pipeline/WritableContainerFactory.java | 2 +-
.../scm/pipeline/WritableECContainerProvider.java | 6 +-
...inerLocationProtocolServerSideTranslatorPB.java | 8 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 4 +-
.../hdds/scm/safemode/DataNodeSafeModeRule.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 16 +--
.../hdds/scm/server/SCMClientProtocolServer.java | 15 +--
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 6 +-
.../hdds/scm/server/StorageContainerManager.java | 12 +--
.../scm/server/StorageContainerManagerStarter.java | 2 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +-
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 2 +-
.../hadoop/hdds/scm/TestHddsServerUtils.java | 35 ++++++-
.../hadoop/hdds/scm/block/TestBlockManager.java | 4 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 6 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 6 +-
.../TestIncrementalContainerReportHandler.java | 4 +-
.../container/balancer/TestContainerBalancer.java | 24 ++---
.../TestSCMContainerPlacementRackAware.java | 2 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 26 ++---
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 2 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 2 +-
.../hdds/scm/node/TestContainerPlacement.java | 2 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 4 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 12 +--
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 2 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 4 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 4 +-
.../hdds/scm/pipeline/TestECPipelineProvider.java | 6 +-
.../TestPipelineDatanodesIntersection.java | 4 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 16 +--
.../scm/pipeline/TestRatisPipelineProvider.java | 4 +-
.../pipeline/TestWritableECContainerProvider.java | 20 ++--
.../TestOneReplicaPipelineSafeModeRule.java | 8 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 8 +-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 6 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 24 ++---
.../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 2 +-
.../testutils/ReplicationNodeManagerMock.java | 2 +-
.../org/apache/ozone/test/LambdaTestUtils.java | 4 +-
.../scm/cli/ContainerBalancerStatusSubcommand.java | 2 +-
.../cli/ReplicationManagerStatusSubcommand.java | 2 +-
.../hdds/scm/cli/SafeModeCheckSubcommand.java | 2 +-
.../hdds/scm/cli/SafeModeExitSubcommand.java | 2 +-
.../scm/cli/datanode/DecommissionSubCommand.java | 2 +-
.../scm/cli/datanode/MaintenanceSubCommand.java | 4 +-
.../scm/cli/datanode/RecommissionSubCommand.java | 2 +-
.../scm/cli/container/TestReportSubCommand.java | 12 +--
.../scm/cli/datanode/TestListInfoSubcommand.java | 6 +-
hadoop-ozone/client/pom.xml | 1 +
.../apache/hadoop/ozone/client/ObjectStore.java | 6 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 14 +--
.../hadoop/ozone/client/OzoneClientFactory.java | 2 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 2 +-
.../apache/hadoop/ozone/client/OzoneVolume.java | 2 +-
.../client/checksum/BaseFileChecksumHelper.java | 9 +-
.../checksum/ReplicatedFileChecksumHelper.java | 6 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 16 ++-
.../hadoop/ozone/client/io/ECBlockInputStream.java | 6 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 10 +-
.../client/io/ECBlockReconstructedInputStream.java | 2 +-
.../io/ECBlockReconstructedStripeInputStream.java | 4 +-
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 5 +-
.../hadoop/ozone/client/rpc/OzoneKMSUtil.java | 4 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 24 ++---
.../hadoop/ozone/client/MockDatanodeStorage.java | 4 +-
.../hadoop/ozone/client/MockOmTransport.java | 4 +-
.../ozone/client/MockXceiverClientFactory.java | 4 +-
.../client/MultiNodePipelineBlockAllocator.java | 2 +-
.../hadoop/ozone/client/TestHddsClientUtils.java | 4 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 8 +-
.../checksum/TestReplicatedFileChecksumHelper.java | 4 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 4 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 8 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 6 +-
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 10 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 1 +
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 2 +-
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 4 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 2 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 4 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 2 +-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 6 +-
.../hadoop/ozone/om/helpers/ServiceInfo.java | 2 +-
.../hadoop/ozone/om/helpers/WithObjectID.java | 2 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 8 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 14 +--
.../hadoop/ozone/security/acl/OzoneObjInfo.java | 2 +-
.../apache/hadoop/ozone/util/OzoneVersionInfo.java | 2 +-
.../org/apache/hadoop/ozone/util/RadixTree.java | 2 +-
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 2 +-
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 2 +-
.../ozone/security/TestGDPRSymmetricKey.java | 2 +-
.../ozone/security/acl/TestOzoneObjInfo.java | 4 +-
.../apache/hadoop/ozone/util/TestRadixTree.java | 4 +-
hadoop-ozone/dev-support/checks/bats.sh | 8 +-
hadoop-ozone/dev-support/checks/checkstyle.sh | 3 +
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/compatibility/.env | 1 +
.../main/compose/compatibility/docker-compose.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 1 +
.../src/main/compose/ozone-csi/docker-compose.yaml | 8 +-
hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 1 +
.../src/main/compose/ozone-ha/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone-mr/hadoop27/.env | 1 +
.../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop31/.env | 1 +
.../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop32/.env | 1 +
.../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-mr/hadoop33/.env | 1 +
.../compose/ozone-mr/hadoop33/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozone-om-ha/.env | 1 +
.../dist/src/main/compose/ozone-om-ha/Dockerfile | 3 +-
.../dist/src/main/compose/ozone-om-prepare/.env | 6 +-
.../compose/ozone-om-prepare/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone-topology/.env | 1 +
.../compose/ozone-topology/docker-compose.yaml | 16 +--
hadoop-ozone/dist/src/main/compose/ozone/.env | 1 +
.../src/main/compose/ozone/docker-compose.yaml | 2 +-
.../dist/src/main/compose/ozone/freon-ockg.yaml | 2 +-
.../dist/src/main/compose/ozone/freon-rk.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 6 --
.../dist/src/main/compose/ozoneblockade/.env | 1 +
.../main/compose/ozoneblockade/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozones3-haproxy/.env | 1 +
.../compose/ozones3-haproxy/docker-compose.yaml | 12 +--
.../dist/src/main/compose/ozonescripts/.env | 1 +
.../dist/src/main/compose/ozonescripts/Dockerfile | 3 +-
.../main/compose/ozonescripts/docker-compose.yaml | 3 +
.../dist/src/main/compose/ozonesecure-ha/.env | 1 +
.../compose/ozonesecure-ha/docker-compose.yaml | 22 ++---
.../dist/src/main/compose/ozonesecure-mr/.env | 1 +
.../compose/ozonesecure-mr/docker-compose.yaml | 8 +-
.../dist/src/main/compose/ozonesecure-mr/test.sh | 3 -
.../dist/src/main/compose/ozonesecure/.env | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 10 +-
hadoop-ozone/dist/src/main/compose/restart/.env | 2 +-
.../src/main/compose/restart/docker-compose.yaml | 2 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 3 +-
.../dist/src/main/compose/upgrade/compose/ha/.env | 1 +
.../main/compose/upgrade/compose/ha/docker-config | 4 +-
.../src/main/compose/upgrade/compose/non-ha/.env | 1 +
hadoop-ozone/dist/src/main/compose/xcompat/.env | 1 +
.../dist/src/main/compose/xcompat/clients.yaml | 2 +-
.../dist/src/main/compose/xcompat/new-cluster.yaml | 2 +-
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 3 +-
.../src/main/smoketest/compatibility/read.robot | 9 +-
.../{kinit.robot => compatibility/setup.robot} | 16 +--
.../src/main/smoketest/compatibility/write.robot | 7 +-
.../dist/src/main/smoketest/mapreduce.robot | 1 +
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 1 +
hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh | 3 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 10 +-
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 2 +-
.../hadoop/ozone/loadgenerators/LoadBucket.java | 2 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 14 +--
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 2 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 42 ++++----
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 4 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 34 +++----
.../fs/ozone/contract/ITestOzoneContractUtils.java | 4 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 4 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 2 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 2 +-
.../TestRatisPipelineCreateAndDestroy.java | 2 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 6 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 6 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +-
.../hadoop/ozone/MiniOzoneClusterProvider.java | 4 +-
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 +-
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +-
.../ozone/TestContainerBalancerOperations.java | 4 +-
.../hadoop/ozone/TestContainerOperations.java | 2 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 +-
.../ozone/client/CertificateClientTestImpl.java | 4 +-
.../hadoop/ozone/client/rpc/TestCommitWatcher.java | 2 +-
.../client/rpc/TestDiscardPreallocatedBlocks.java | 2 +-
.../ozone/client/rpc/TestECKeyOutputStream.java | 2 +-
.../client/rpc/TestFailureHandlingByClient.java | 2 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 10 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 10 +-
.../rpc/TestOzoneClientRetriesOnExceptions.java | 4 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 60 ++++++------
.../rpc/TestOzoneRpcClientForAclAuditLog.java | 18 ++--
.../TestOzoneRpcClientWithKeyLatestVersion.java | 2 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 2 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 6 +-
.../ozone/client/rpc/TestWatchForCommit.java | 2 +-
.../ozone/client/rpc/read/ECStreamTestUtil.java | 4 +-
.../rpc/read/TestBlockInputStreamFactoryImpl.java | 4 +-
.../client/rpc/read/TestECBlockInputStream.java | 4 +-
.../rpc/read/TestECBlockInputStreamProxy.java | 6 +-
.../read/TestECBlockReconstructedInputStream.java | 26 ++---
.../TestECBlockReconstructedStripeInputStream.java | 2 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 2 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 10 +-
.../TestCloseContainerByPipeline.java | 2 +-
.../commandhandler/TestDeleteContainerHandler.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 4 +-
.../container/ozoneimpl/TestOzoneContainer.java | 2 +-
.../container/server/TestContainerServer.java | 2 +-
.../server/TestSecureContainerServer.java | 6 +-
.../TestDatanodeHddsVolumeFailureDetection.java | 2 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 26 ++---
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 14 +--
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 +-
.../ozone/om/TestOmStartupSlvLessThanMlv.java | 2 +-
.../ozone/om/TestOzoneManagerConfiguration.java | 4 +-
.../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 8 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 2 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 2 +-
.../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +-
.../ozone/recon/TestReconWithOzoneManager.java | 11 ++-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 8 +-
.../hadoop/ozone/scm/TestAllocateContainer.java | 2 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 4 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 6 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 10 +-
.../scm/node/TestDecommissionAndMaintenance.java | 18 ++--
.../hadoop/ozone/scm/node/TestQueryNode.java | 2 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 22 +++--
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 2 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 58 +++++------
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 8 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 22 ++---
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 2 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 12 +--
.../hadoop/ozone/om/OzoneManagerPrepareState.java | 4 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 2 +-
.../hadoop/ozone/om/S3SecretManagerImpl.java | 2 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 12 +--
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 12 +--
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 4 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 6 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 6 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 6 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 2 +-
.../om/request/file/OMDirectoryCreateRequest.java | 2 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 2 +-
.../ozone/om/request/file/OMFileRequest.java | 10 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 4 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 2 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 2 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 2 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 8 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +-
.../multipart/S3MultipartUploadAbortRequest.java | 2 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 2 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 2 +-
.../volume/acl/OMVolumeRemoveAclRequest.java | 2 +-
.../request/volume/acl/OMVolumeSetAclRequest.java | 2 +-
.../OzoneDelegationTokenSecretManager.java | 2 +-
.../hadoop/ozone/security/OzoneSecretStore.java | 4 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 2 +-
.../apache/hadoop/ozone/om/TestOMDBDefinition.java | 2 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 14 +--
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +-
...TestOzoneManagerDoubleBufferWithOMResponse.java | 4 +-
.../om/ratis/TestOzoneManagerRatisServer.java | 2 +-
.../om/ratis/TestOzoneManagerStateMachine.java | 2 +-
.../request/file/TestOMDirectoryCreateRequest.java | 2 +-
.../om/request/file/TestOMFileCreateRequest.java | 6 +-
.../om/request/key/TestOMKeyCommitRequest.java | 4 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 2 +-
.../om/request/key/TestOMKeyCreateRequest.java | 2 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 4 +-
.../key/TestOMKeyPurgeRequestAndResponse.java | 2 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 2 +-
.../TestS3MultipartUploadCompleteRequest.java | 2 +-
.../request/volume/TestOMVolumeCreateRequest.java | 2 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 2 +-
.../ozone/om/response/TestCleanupTableInfo.java | 4 +-
.../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 2 +-
.../om/upgrade/TestOzoneManagerPrepareState.java | 2 +-
.../TestOzoneDelegationTokenSecretManager.java | 6 +-
.../ozone/security/TestOzoneTokenIdentifier.java | 6 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 6 +-
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 16 ++-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 9 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 18 ++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 3 +-
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 +
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 24 ++++-
.../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 2 +-
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 35 +++++++
hadoop-ozone/ozonefs-shaded/pom.xml | 2 +-
.../org/hadoop/ozone/recon/codegen/SqlDbUtils.java | 4 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 7 ++
.../ozone/recon/api/MetricsProxyEndpoint.java | 4 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 2 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 2 +-
.../hadoop/ozone/recon/codec/NSSummaryCodec.java | 2 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 4 +-
.../ozone/recon/scm/ReconContainerManager.java | 4 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 2 +-
.../scm/ReconStorageContainerManagerFacade.java | 4 +-
.../recon/spi/impl/ContainerKeyPrefixCodec.java | 2 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 62 ++++++++++--
.../ozone/recon/tasks/FileSizeCountTask.java | 4 +-
.../hadoop/ozone/recon/tasks/TableCountTask.java | 2 +-
.../ozone/recon/api/TestTaskStatusService.java | 2 +-
.../ozone/recon/fsck/TestContainerHealthTask.java | 2 +-
.../TestUtilizationSchemaDefinition.java | 2 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 88 ++++++++++++++++-
.../recon/tasks/TestContainerKeyMapperTask.java | 2 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 4 +-
.../ozone/recon/tasks/TestTableCountTask.java | 2 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 17 ++--
.../hadoop/ozone/s3/VirtualHostStyleFilter.java | 8 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 58 +++++------
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 18 ++--
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 108 +++++++++------------
.../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 6 +-
.../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 4 +-
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 20 +++-
.../hadoop/ozone/s3/signature/Credential.java | 2 +-
.../apache/hadoop/ozone/s3/util/ContinueToken.java | 2 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 39 +++++++-
.../ozone/s3/TestVirtualHostStyleFilter.java | 2 +-
.../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +-
.../hadoop/ozone/s3/endpoint/TestRootList.java | 2 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 6 +-
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 6 +-
.../admin/scm/FinalizeUpgradeCommandUtil.java | 4 +-
.../hadoop/ozone/audit/parser/AuditParser.java | 2 +-
.../ozone/audit/parser/common/DatabaseHelper.java | 16 +--
.../parser/handler/TemplateCommandHandler.java | 2 +-
.../ozone/audit/parser/model/AuditEntry.java | 22 ++---
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 6 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 8 +-
.../apache/hadoop/ozone/debug/PrefixParser.java | 2 +-
.../apache/hadoop/ozone/debug/ReadReplicas.java | 14 +--
.../hadoop/ozone/freon/BaseFreonGenerator.java | 2 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 12 +--
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 4 +-
.../hadoop/ozone/freon/HadoopFsGenerator.java | 2 +-
.../hadoop/ozone/freon/StreamingGenerator.java | 2 +-
.../GenerateOzoneRequiredConfigurations.java | 2 +-
.../apache/hadoop/ozone/shell/OzoneAddress.java | 4 +-
.../ozone/shell/bucket/CreateBucketHandler.java | 4 +-
.../hadoop/ozone/audit/parser/TestAuditParser.java | 2 +-
.../hadoop/ozone/conf/TestGetConfOptions.java | 4 +-
.../TestGenerateOzoneRequiredConfigurations.java | 4 +-
.../org/apache/hadoop/test/OzoneTestDriver.java | 6 +-
pom.xml | 26 ++++-
559 files changed, 2169 insertions(+), 1658 deletions(-)
copy hadoop-ozone/dist/src/main/smoketest/{kinit.robot =>
compatibility/setup.robot} (75%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
