This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 8d1d957 HDDS-6022. [Multi-Tenant] Implement DeleteTenant: `ozone
tenant delete` (#2857)
add 605a734 HDDS-5697. SCMContainerPlacementRackAware should both
affinity nodes before fallback (#2597)
add e5c647e HDDS-5370. [FSO] Handle OMClientRequest based on the bucket
layout. (#2533)
add 181266b HDDS-5777. Provide an option to dump table scan data to file.
(#2675)
add 4b3502f HDDS-5792. Speed up TestNSSummaryAdmin by having zero
Datanodes (#2688)
add 2e3c46c HDDS-5782. DatanodeRatisGrpcConfig use wrong type for config
keys. (#2678)
add 041684f HDDS-5789. Enabling container tokens on a upgraded cluster
fails SCM to start up. (#2691)
add af7dc7e HDDS-5800. Bucket usedByte was incorrect when deleting
duplicate keys.
add 8cc6719 HDDS-5523. Fix multipart upload failure in s3 compatibility
tests (#2509)
add aa9eade HDDS-5757. balancer should stop when the cluster can not be
balanced any more (#2676)
add e2c3ac3 HDDS-5747. Reset default values in
ContainerBalancerConfiguration (#2657)
add 5c2ac71 HDDS-5788. Reduce run time for TestOzoneManagerHA tests
(#2689)
add af5b48e HDDS-5636. Use bucket layout specific DB table in
OmMetadataManager:getOpenKeyTable(). (#2561)
add 093f972 HDDS-5787. Provide capability in Ozone to enable ratis
pre-vote (#2680)
add 05eb172 HDDS-5762. remove piplelineStateManager V1 code (#2661)
add c4cbcad HDDS-5710. initialize sequenceIdToLastIdMap when
SequenceIdGenerator#StateManager reinitializes (#2611)
add 53a0009 HDDS-5793. Speed up TestBlockOutputStreamWithFailures by
combining test cases (#2690)
add 1da0bd8 HDDS-5609. Document CI workflow (#2607)
add c886a2f HDDS-5797. Support setting Datanode Reserved Space in
MiniOzoneCluster. (#2695)
add 7bdc1bb HDDS-5799. Speed up TestOzoneFileSystem by using shorter
trash interval (#2699)
add 1ea1d6c HDDS-5808. Update commons-io to 2.11.0 (#2705)
add 186cd6e HDDS-5817. Extract properties for some dependency versions
(#2710)
add 20e8429 HDDS-5815 FileEncryptionInfo is not necessary to be
referenced inside KeyOutputStream. (#2708)
add a59c501 HDDS-5830. Increase idea.max.intellisense.filesize (#2715)
add 8d885b4 HDDS-5557. Cleanup
OzoneBlockTokenSecretManager#validateToken. (#2711)
add ea81ef8 HDDS-5816 Rearrange code and refactor some logic into new
methods in prep for EC addition. (#2709)
add 8eaa9c5 HDDS-4440. Diagrams for old and new approach are mixed up
(#2555)
add c5a92a2 HDDS-5824. `ozone sh volume/bucket/key list` should print
valid JSON array (#2713)
add db97a32 HDDS-5733. Incorrect calculation of iteration related metrics
in ContainerBalancer (#2631)
add efb5263 HDDS-5806. remove BackgroundPipelineCreator v1 code (#2700)
add 57729b4 HDDS-5536. Add metrics for ReplicationSupervisor (#2493)
add 1ffcfe9 HDDS-5349. Avoid usage of locks in listStatus. (#2369)
add 3e05e71 HDDS-5373. [FSO] Define default bucket layout. (#2707)
add f902b13 HDDS-5785. containerCreationLock is a severe bottleneck when
there are many containers to create. (#2679)
add a62c61f HDDS-5823. Change the key permission to 600. (#2722)
add 2f49993 HDDS-5845. Add metrics for stateMachineDataCache cache hit
and miss. (#2726)
add 730a2e8 HDDS-5842. Parse and dump SCM ratis segment file to printable
text (#2731)
add e19fa38 HDDS-5814. Handle InterruptedException in Ozone (#2706)
add ae812c06 HDDS-5831. Remove empty TaskQueue in ContainerStateMachine.
(#2716)
add ed069bd HDDS-5854. Bump json-smart to 2.3.1 (#2736)
add 7376cfe HDDS-5855. Bump commons-compress to 1.21 (#2737)
add 0e5ad31 HDDS-5856. Bump Spring to 5.2.17 (#2738)
add d341c05 HDDS-5857. Bump aws-java-sdk to latest 1.11 (#2740)
add 24ddf0f HDDS-5659. Use bucket layout specific DB table in
OmMetadataManager.getKeyTable() - part1 (#2724)
add bf6da11 HDDS-5858. Fix resource leak due to Files.list (#2741)
add f2f6aa5 HDDS-5671. ContainerBalancer#stop should prevent the current
balancing thread from interrupting itself. (#2578)
add bc81dcf HDDS-5734. add a command to trigger datanode executing du
immediately (#2685)
add a1701e5 HDDS-5850. Update default value of
'ozone.scm.ha.ratis.segment.size' and 'preallocated.size' to improve SCM write
perf. (#2732)
add 7d79d73 HDDS-5812. Increase Ratis version to 2.2.0 (#2753)
add fbb8cf0 HDDS-5794. The misleading "No available thread in pool for
past * second" log message in DN StateContext (#2693)
add 996518a HDDS-5860. Shade Jackson for Ozone Filesystem (#2744)
add 42d3de3 HDDS-5725. Catch possible AlreadyExistsException for create
pipeline command. (#2623)
add 2a4fcde HDDS-5888. Handle InterruptedException in OzoneClientProducer
and ContainerBalancer (#2761)
add ba038e4 HDDS-5875. Do not cache tokens as part of Key Information.
(#2754)
add 2a4bc83 HDDS-5461. Move old objects to delete table on overwrite
(#2433)
add 6e5db67 HDDS-5871. Fix ContainerBalancerConfiguration annotation
string (#2762)
add d8d42b5 HDDS-5868. Change getBucketInfo annotation (#2749)
add 0828afc HDDS-5843. SCM terminates when adding container to a pipeline
during startup (#2745)
add c5aaad8 HDDS-3371. Cleanup of old write-path of key in OM (#2703)
add fc666fa HDDS-5893. Potential data loss on multipart upload commit
(#2763)
add a51822f HDDS-5882 Changes to protobuf messages for HDDS-5881 (#2774)
add f133303 HDDS-5707. List container supports replication factor filter.
(#2606)
add 990db75 HDDS-5835. Follow up task to use bucket layout specific DB
table in OmMetadataManager.getKeyTable() - part2 (#2751)
add 0906337 HDDS-5902 Ambiguous OM kerberos keytab settings (#2769)
add a5d6707 HDDS-5901. Delete on volume/bucket throws fatal error on ofs
(#2776)
add 26824ca HDDS-5884. OM Validate S3 Auth for write requests. (#2778)
add d018d00 HDDS-5805. remove containerStateManager V1 code (#2727)
add f153236 HDDS-5863. Error message having null fields on volume
creation (#2752)
add 5c76856 HDDS-5885. OM Auth Validate for read requests and handle non
ratis enabled code. (#2779)
add 5f0a588 HDDS-5908. MPU getKey can fail, if completeMPU result is
still in cache. (#2787)
add e427b71 HDDS-5883 Change S3G client to set S3 Auth per req (#2775)
add 73a081a HDDS-5864. Retry when DN connection issue during
getBlock/ReadChunk call during Ozone key Read (#2746)
add 83f8b04 HDDS-5736. Update navbar from Hadoop to Ozone on doc page
(#2792)
add 47cb26a HDDS-3983 Ozone RocksDB Iterator wrapper should not expose
key() and value() API. (#2402)
add 565972c Support more detailed error log when handleFlush in Ozone
client (#2795)
add 1cf43ac HDDS-5931. Broken link in dist/README (#2799)
add 9cea16d HDDS-5933. Ozone Filesystem shaded jars include unnecessary
dependencies (#2798)
add eb7136d HDDS-3369. Cleanup old write-path of volume in OM. (#2780)
add c088cd1 HDDS-5925. Fix SecuringOzoneHTTP doc (#2796)
add ac86851 HDDS-5943. Shutdown ResultHandlerExecutorService for
StorageVolumeChecker. (#2806)
add aa7e083 HDDS-5944. Fix typo in OzoneManagerServiceProviderImpl.java
(#2807)
add 6ebb316 HDDS-5945. bump rocksdb version to 6.25.3 (#2809)
add 4c2181f HDDS-5935. Bump Spring to 5.2.18 (#2801)
add f3b482a HDDS-5922. ignore deletetransaction when container is not
found (#2793)
add c63479c HDDS-5839. Make sure buckets created from OFS are in
FILE_SYSTEM_OPTIMIZED layout (#2730)
add 96e2d7f HDDS-5947. Remove unused mina-core and sshd-core dependencies
(#2811)
add 252fee4 HDDS-5910 Add additional verification for S3 Auth. (#2817)
add 9d9e780 HDDS-5956. Speed up
TestOzoneRpcClientAbstract#testZReadKeyWithUnhealthyContainerReplica (#2820)
add 0eedab9 HDDS-5929. Make FSO and OBS bucket layouts independent of
normalization config flag (#2819)
add ba3ac30 HDDS-5960. Change option name to bucketlayout instead of type
(#2825)
add 4558e40 HDDS-5970. Remove OMKeyRequest#getBucketLayout overridden
method in subclasses (#2830)
add 9384bd7 HDDS-5820. Intermittent failure in
TestPipelinePlacementPolicy#testPickLowestLoadAnchor (#2757)
add d0e9140 HDDS-5958. Refine container scrub log message. (#2823)
add 45c1899 HDDS-5937. Inaccurate bucket info returned from bucket list
command in Shell (#2832)
add c2db377 HDDS-5872. Do not failover on some RpcExceptions (#2772)
add 71f47d4 HDDS-5981. Refactor usage of bucket type. (#2836)
add 026d878 HDDS-5982. Remove isBucketFSOptimized flag. (#2838)
add 346b55d HDDS-5906. Fix DBScanner to support Datanode DB (#2781)
add 1b7072c HDDS-5983. Prefix Parser tool should only work for FSO
buckets. (#2839)
add 5ef59b4 HDDS-5957. Bump proto-backwards-compatibility from 1.0.5 to
1.0.7 (#2821)
add b4a785c HDDS-5891. OFS mkdir -p does not work as expected for bucket
creation when volume exists due to volume create ACL check (#2814)
add 6ea9916 HDDS-5964. [S3-TDE] Range Reads not working on encrypted
buckets (#2827)
add 83e5aa3 HDDS-5848. Introduce more replication metrics (#2758)
add 4348d1d HDDS-5980. Get on a key which is created/overwrite through
MPU on a TDE bucket, the data does not match with uploaded data. (#2844)
add 78e725c HDDS-5969. TestRootedOzoneFileSystem parameterized test is
not initialized properly (#2843)
add 281f3b0 HDDS-5870. Make Datanode usageinfo command output more
readable (#2816)
add 07b54e2 HDDS-5534. Verify config is updated on all OMs before
proceeding with Bootstrap (#2491)
add 722fd59 HDDS-5898 S3G in secure mode checks OM version. (#2822)
add 52e619c HDDS-5517. Support multiple container moves from a source
datanode in one balance iteration (#2808)
add 6372776 HDDS-5976. add raftlog IOException notifier for SCM HA (#2840)
add bc6029e [FSO] Allow existing/old buckets with any layout during OM
startup (#2571)
add ffe53e7 HDDS-6019. Fix s3g create key error in cluster with
replication one. (#2855)
add f780851 HDDS-6003. Update Ozone master branch version after 1.2.0
release (#2863)
add 4f56b1a HDDS-6043. Buckets created via link command do not mirror
layout of s… (#2867)
add 7afd64f HDDS-5897. Support configuration for including/excluding
datanodes for balancing (#2786)
add 97d900e HDDS-5989. Binary content printed for failed chunk write
(#2846)
add 1e76ffc HDDS-6011. Freon datanode chunk generator fails with NPE in
secure cluster (#2853)
add 5472ea4 HDDS-5497. Save log from init containers (#2547)
add 8b4d4a9 HDDS-5804. SCMContainerPlacementCapacity to choose lower
utilized node (#2784)
add 4e91226 HDDS-5917. S3 acceptance test failure due to too wide
assertion (#2789)
add 5b0604d HDDS-5702. SCM remote benchmark tool. (#2600)
add 2341c7c HDDS-5936. Duplicate Test Ozone Shell invocation (#2803)
add 1892230 HDDS-6054. Fix resource leak due to OzoneManager.createOm
(#2876)
add ec77f6c HDDS-6000. Freon datanode chunk validator fails checksum
validation (#2850)
add 090672d HDDS-6055. Refactor isFileSystemOptimized usage in
OzoneManagerUtils (#2877)
add b69a3d1 HDDS-5959. Handles bucket layout validation logic in ofs/o3fs
client. (#2837)
add b8e1e8f HDDS-5999. Freon datanode chunk validator does not find
pipeline from param (#2879)
add 3012cd1 HDDS-6014. Add proto.lock files from ozone-1.2 release branch
to master (#2865)
add ef1f3d3 HDDS-5962. Limit grpc threads in XceiverServerGrpc. (#2824)
add d6e0e47 HDDS-5997 Centralize string based replication config
validation via ReplicationConfigValidator (#2871)
add 8ef63ec HDDS-6058. Bump aws-java-sdk to latest 1.12 (#2881)
add 66c581d HDDS-6041. [FSO] Revisit robot test cases related to
prefix/simple me… (#2885)
add ab3b731 HDDS-6061. Peer datanode cannot add group for pipeline in
secure env (#2882)
add e86119f HDDS-5965. Recon should be able to distinguish between
containers that have no replicas and those have all replicas as UNHEALTHY.
(#2874)
add 9c4bb72 HDDS-5973. Changed sequence of steps during pipeline close.
(#2847)
add df0489a HDDS-5903. Add Support for Bucket Owner Acls (#2826)
add 3097d2b HDDS-6059. Change Cannot close container log message from
WARN to DEBUG (#2890)
add ea53dc1 HDDS-3227. Ensure eviction of stateMachineData from cache
only when both followers catch up (#2704)
add 46c95ba HDDS-6067. Improve Debugging around unhealthy container state
on Datanode. (#2888)
add 937a3d0 HDDS-6071. ResourceLimitCache leaks permits (#2895)
add 176ef94 HDDS-5385. [FSO] Remove ozone.om.metadata.layout config in OM
(#2887)
add ab8cc32 HDDS-6073. Document: Fix images alignment in the doc pages
(#2898)
add d972c7b HDDS-6069: Fix XmlRootElement for CopyObjectResponse (#2891)
add 41f6844 HDDS-6046. Multi-threaded validator for freon
RandomKeyGenerator. (#2869)
add 8532e08 HDDS-6078. Move BucketManagerImpl#resolveLinkBucketLayout
into utilit… (#2902)
add 831b195 HDDS-5974. Remove OMClientResponse#getBucketLayout override
in subclasses - part1 (#2894)
add 173f46a HDDS-6085. Update log4j to 2.15 (#2906)
new f5581ce Merge remote-tracking branch 'asf/master' into HDDS-4944
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/ci.md | 102 ++
CONTRIBUTING.md | 4 +-
hadoop-hdds/client/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 38 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 52 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 +
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 14 +-
.../hdds/scm/storage/TestBlockInputStream.java | 32 +-
hadoop-hdds/common/pom.xml | 4 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 122 ++
.../java/org/apache/hadoop/hdds/NodeDetails.java | 0
.../hadoop/hdds/client/RatisReplicationConfig.java | 27 +-
.../hdds/client/ReplicatedReplicationConfig.java} | 20 +-
.../hadoop/hdds/client/ReplicationConfig.java | 138 +-
.../hdds/client/ReplicationConfigValidator.java | 5 +-
.../hdds/client/StandaloneReplicationConfig.java | 23 +-
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 23 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 1 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 6 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 7 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 31 +-
.../hadoop/hdds/scm/net/NetworkTopology.java | 3 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 5 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 2 +-
.../protocol/StorageContainerLocationProtocol.java | 20 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 5 -
.../hadoop/hdds/utils/ResourceLimitCache.java | 7 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 12 +
.../common/src/main/resources/ozone-default.xml | 94 +-
.../hadoop/hdds/client/TestReplicationConfig.java | 280 ++--
.../hadoop/hdds/utils/TestResourceLimitCache.java | 46 +-
hadoop-hdds/config/pom.xml | 4 +-
hadoop-hdds/container-service/pom.xml | 4 +-
.../container/common/helpers/ContainerUtils.java | 68 -
.../container/common/impl/HddsDispatcher.java | 4 +-
.../common/statemachine/DatanodeConfiguration.java | 43 +
.../common/statemachine/DatanodeStateMachine.java | 13 +-
.../common/statemachine/StateContext.java | 14 +-
.../CreatePipelineCommandHandler.java | 10 +-
....java => RefreshVolumeUsageCommandHandler.java} | 67 +-
.../common/transport/server/XceiverServerGrpc.java | 52 +-
.../common/transport/server/ratis/CSMMetrics.java | 10 +
.../server/ratis/ContainerStateMachine.java | 82 +-
.../transport/server/ratis/XceiverServerRatis.java | 12 +-
.../container/common/volume/MutableVolumeSet.java | 4 +
.../container/common/volume/StorageVolume.java | 4 +
.../common/volume/StorageVolumeChecker.java | 1 +
.../ozone/container/common/volume/VolumeInfo.java | 13 +-
.../ozone/container/common/volume/VolumeUsage.java | 3 +
.../container/keyvalue/KeyValueContainer.java | 4 +
.../container/keyvalue/KeyValueContainerCheck.java | 2 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 29 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 6 +-
.../replication/DownloadAndImportReplicator.java | 5 +-
.../container/replication/MeasuredReplicator.java | 34 +-
.../replication/ReplicationSupervisor.java | 9 +-
.../replication/ReplicationSupervisorMetrics.java | 71 +
.../replication/SimpleContainerDownloader.java | 7 +-
.../common/helpers/TestContainerUtils.java | 2 +-
.../common/volume/TestStorageVolumeChecker.java | 4 +
.../replication/TestMeasuredReplicator.java | 34 +-
.../replication/TestReplicationSupervisor.java | 59 +-
hadoop-hdds/docs/content/concept/Containers.md | 2 +-
hadoop-hdds/docs/content/concept/Datanodes.md | 4 +-
hadoop-hdds/docs/content/concept/OzoneManager.md | 6 +-
hadoop-hdds/docs/content/concept/Recon.md | 6 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 5 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 24 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +-
hadoop-hdds/docs/content/feature/Topology.md | 2 +-
hadoop-hdds/docs/content/interface/CSI.md | 4 +-
hadoop-hdds/docs/content/interface/CSI.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Cli.md | 7 +-
.../docs/content/security/SecuringOzoneHTTP.md | 20 +-
hadoop-hdds/docs/content/start/FromSource.md | 2 +-
hadoop-hdds/docs/pom.xml | 4 +-
.../docs/static/design/s3-performance-new.png | Bin 33688 -> 34927 bytes
.../docs/static/design/s3-performance-old.png | Bin 34927 -> 33688 bytes
.../themes/ozonedoc/layouts/partials/navbar.html | 6 +-
.../themes/ozonedoc/layouts/partials/sidebar.html | 4 +-
.../ozonedoc/layouts/shortcodes/image.html} | 7 +-
hadoop-hdds/framework/pom.xml | 4 +-
.../java/org/apache/hadoop/hdds/ExitManager.java | 12 +-
.../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java | 4 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 16 +
.../scm/protocol/ScmBlockLocationProtocol.java | 2 +-
...inerLocationProtocolClientSideTranslatorPB.java | 12 +-
.../certificate/client/DNCertificateClient.java | 5 -
.../client/DefaultCertificateClient.java | 2 +-
.../certificate/client/OMCertificateClient.java | 5 -
.../certificate/client/SCMCertificateClient.java | 10 +-
.../hadoop/hdds/security/x509/keys/KeyCodec.java | 31 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 16 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 19 +-
.../apache/hadoop/hdds/utils/db/TableIterator.java | 12 -
.../apache/hadoop/hdds/utils/db/TypedTable.java | 18 -
.../hdds/security/x509/keys/TestKeyCodec.java | 5 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 19 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
.../src/main/proto/ScmAdminProtocol.proto | 1 +
.../interface-admin/src/main/resources/proto.lock | 504 +++++++
hadoop-hdds/interface-client/pom.xml | 4 +-
.../interface-client/src/main/proto/hdds.proto | 11 +
.../interface-client/src/main/resources/proto.lock | 408 +++++-
hadoop-hdds/interface-server/pom.xml | 4 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../interface-server/src/main/resources/proto.lock | 1024 +++++++++++++++
hadoop-hdds/pom.xml | 4 +-
hadoop-hdds/server-scm/pom.xml | 4 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 10 -
.../scm/container/CloseContainerEventHandler.java | 12 +-
.../hdds/scm/container/ContainerManager.java | 2 +-
.../hdds/scm/container/ContainerManagerImpl.java | 4 +-
.../hdds/scm/container/ContainerStateManager.java | 570 ++------
.../scm/container/ContainerStateManagerImpl.java | 19 +-
.../scm/container/ContainerStateManagerV2.java | 189 ---
.../hdds/scm/container/ReplicationManager.java | 43 +-
.../scm/container/balancer/ContainerBalancer.java | 338 ++---
.../balancer/ContainerBalancerConfiguration.java | 125 +-
.../balancer/ContainerBalancerMetrics.java | 115 +-
.../ContainerBalancerSelectionCriteria.java | 22 +-
.../scm/container/balancer/FindSourceGreedy.java | 158 +++
.../scm/container/balancer/FindSourceStrategy.java | 67 +
.../scm/container/balancer/FindTargetGreedy.java | 121 +-
.../scm/container/balancer/FindTargetStrategy.java | 32 +-
.../algorithms/SCMContainerPlacementCapacity.java | 2 +-
.../algorithms/SCMContainerPlacementRackAware.java | 72 +-
.../replication/ReplicationManagerMetrics.java | 41 +
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 11 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 23 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 24 +
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 7 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 26 +-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 30 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 8 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 13 +
.../scm/pipeline/BackgroundPipelineCreator.java | 266 +++-
.../scm/pipeline/BackgroundPipelineCreatorV2.java | 332 -----
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 2 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 10 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 54 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 6 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 271 ++--
...erV2Impl.java => PipelineStateManagerImpl.java} | 70 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 27 +
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
.../hadoop/hdds/scm/pipeline/StateManager.java | 127 --
.../algorithms/DefaultLeaderChoosePolicy.java | 4 +-
.../choose/algorithms/LeaderChoosePolicy.java | 8 +-
.../algorithms/LeaderChoosePolicyFactory.java | 8 +-
.../algorithms/MinLeaderCountChoosePolicy.java | 6 +-
...lockLocationProtocolServerSideTranslatorPB.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 6 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 51 +-
.../hdds/scm/server/StorageContainerManager.java | 26 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 5 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 18 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 18 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 5 +
.../scm/container/TestContainerManagerImpl.java | 6 +-
.../scm/container/TestContainerReportHandler.java | 267 ++--
.../scm/container/TestContainerStateManager.java | 91 +-
.../TestIncrementalContainerReportHandler.java | 126 +-
.../hdds/scm/container/TestReplicationManager.java | 380 +++---
.../scm/container/TestUnknownContainerReport.java | 47 +-
.../container/balancer/TestContainerBalancer.java | 146 ++-
.../hdds/scm/container/balancer}/package-info.java | 7 +-
.../TestSCMContainerPlacementCapacity.java | 6 +-
.../TestSCMContainerPlacementRackAware.java | 15 +
.../hdds/scm/ha/TestReplicationAnnotation.java | 8 +-
.../hdds/scm/node/TestContainerPlacement.java | 31 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 6 +
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 11 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 33 +-
.../scm/pipeline/MockRatisPipelineProvider.java | 8 +-
.../TestPipelineDatanodesIntersection.java | 52 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 85 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 73 +-
...ager.java => TestPipelineStateManagerImpl.java} | 245 ++--
.../scm/pipeline/TestRatisPipelineProvider.java | 66 +-
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 209 ---
.../scm/pipeline/TestSimplePipelineProvider.java | 48 +-
.../choose/algorithms/TestLeaderChoosePolicy.java | 6 +-
.../placement/TestContainerPlacement.java | 4 +-
.../testutils/ReplicationNodeManagerMock.java | 11 +
hadoop-hdds/test-utils/pom.xml | 4 +-
hadoop-hdds/tools/pom.xml | 4 +-
.../hdds/scm/cli/ContainerOperationClient.java | 5 +-
.../hdds/scm/cli/container/ListSubcommand.java | 6 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 41 +-
hadoop-ozone/client/pom.xml | 4 +-
.../org/apache/hadoop/ozone/client/BucketArgs.java | 29 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 40 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 268 ++--
.../client/io/BlockOutputStreamEntryPool.java | 109 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 16 +-
.../hadoop/ozone/client/io/OzoneInputStream.java | 33 +-
.../ozone/client/protocol/ClientProtocol.java | 20 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 113 +-
.../hadoop/ozone/client/rpc/RpcClientTest.java | 217 ++++
hadoop-ozone/common/pom.xml | 12 +-
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 3 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 109 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 34 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 11 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 25 +
.../hadoop/ozone/om/helpers/OMNodeDetails.java | 50 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 32 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 27 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 24 +-
.../hadoop/ozone/om/helpers/ServiceInfo.java | 41 +-
.../MultiTenantAccessAuthorizerRangerPlugin.java | 2 +-
...erServiceProtocol.java => OMAdminProtocol.java} | 9 +-
.../hadoop/ozone/om/protocol/OMConfiguration.java | 92 ++
.../ozone/om/protocol/OzoneManagerProtocol.java | 224 +++-
.../OmDeleteKeys.java => protocol/S3Auth.java} | 43 +-
.../protocolPB/OMAdminProtocolClientSideImpl.java | 136 ++
...rviceProtocolPB.java => OMAdminProtocolPB.java} | 9 +-
...nsport.java => OzoneManagerClientProtocol.java} | 33 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 63 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 41 +-
hadoop-ozone/csi/pom.xml | 4 +-
.../org/apache/hadoop/ozone/csi/NodeService.java | 10 +-
hadoop-ozone/datanode/pom.xml | 4 +-
hadoop-ozone/dist/README.md | 6 +-
hadoop-ozone/dist/pom.xml | 4 +-
hadoop-ozone/dist/src/main/compose/ozone/README.md | 6 -
.../src/main/compose/ozone/docker-compose.yaml | 10 -
hadoop-ozone/dist/src/main/compose/ozone/run.sh | 4 -
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 29 +-
.../src/main/compose/ozonesecure-mr/docker-config | 3 +
.../dist/src/main/compose/xcompat/docker-config | 1 +
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 5 +-
.../src/main/smoketest/admincli/container.robot | 8 +
.../dist/src/main/smoketest/basic/links.robot | 8 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 10 +-
.../main/smoketest/basic/ozone-shell-single.robot | 2 +-
.../src/main/smoketest/basic/ozone-shell.robot | 15 +-
.../dist/src/main/smoketest/createmrenv.robot | 2 +-
.../dist/src/main/smoketest/freon/generate.robot | 9 +
.../dist/src/main/smoketest/freon/validate.robot | 9 +
.../dist/src/main/smoketest/omha/testOMHA.robot | 2 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 18 +-
.../dist/src/main/smoketest/ozonefs/setup.robot | 12 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 23 +-
.../dist/src/main/smoketest/s3/objectdelete.robot | 6 +-
.../main/smoketest/security/ozone-secure-fs.robot | 12 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 4 +-
.../dev-support/findbugsExcludeFile.xml | 4 +
hadoop-ozone/integration-test/pom.xml | 4 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 19 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 4 +
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 3 +
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 85 +-
.../fs/ozone/TestOzoneFileInterfacesWithFSO.java | 25 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 47 +-
.../fs/ozone/TestOzoneFileSystemMetrics.java | 3 +
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 2 +
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 9 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 9 +-
.../fs/ozone/TestOzoneFileSystemWithLinks.java | 248 ++++
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 5 +
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 161 ++-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 12 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 5 +-
.../TestContainerStateManagerIntegration.java | 2 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 4 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 19 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 7 +
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 150 ++-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 16 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 5 +
.../hadoop/ozone/TestStandardOutputUtil.java | 84 ++
.../rpc/TestBlockOutputStreamWithFailures.java | 20 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 20 +-
.../rpc/TestContainerStateMachineFailures.java | 20 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 11 +
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 31 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 140 +-
.../hadoop/ozone/client/rpc/TestReadRetries.java | 14 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 100 +-
.../commandhandler/TestBlockDeletion.java | 10 +-
.../ozone/freon/TestHadoopDirTreeGenerator.java | 5 +-
.../freon/TestHadoopDirTreeGeneratorWithFSO.java | 4 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 3 +
.../apache/hadoop/ozone/om/TestBucketOwner.java | 244 ++++
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 44 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 10 +-
.../ozone/om/TestOMStartupWithBucketLayout.java | 161 +++
.../apache/hadoop/ozone/om/TestObjectStore.java | 233 ++++
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 26 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 6 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 27 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 64 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 273 ++--
.../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 184 ++-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 113 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 22 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 98 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 65 +
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 28 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 3 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 71 +
.../TestOzoneHARatisLogParser.java} | 52 +-
.../ozone/recon/TestReconWithOzoneManager.java | 19 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 6 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 123 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 141 +-
hadoop-ozone/interface-client/pom.xml | 4 +-
...ServiceProtocol.proto => OMAdminProtocol.proto} | 43 +-
.../src/main/proto/OmClientProtocol.proto | 59 +-
.../interface-client/src/main/resources/proto.lock | 826 +++++++++++-
hadoop-ozone/interface-storage/pom.xml | 4 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 6 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 5 +-
hadoop-ozone/ozone-manager/pom.xml | 4 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 26 +-
.../hadoop/ozone/om/DirectoryDeletingService.java | 21 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 779 +++++++----
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 14 +
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 5 +-
.../apache/hadoop/ozone/om/OMStarterInterface.java | 2 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 29 +-
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 125 ++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 1374 ++++++--------------
.../hadoop/ozone/om/OzoneManagerStarter.java | 59 +-
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 180 +++
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 7 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 1 +
.../org/apache/hadoop/ozone/om/VolumeManager.java | 40 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 300 -----
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 6 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 42 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 185 +--
.../hadoop/ozone/om/request/OMClientRequest.java | 114 +-
.../ozone/om/request/OMKeyRequestFactory.java | 139 ++
.../om/request/bucket/OMBucketCreateRequest.java | 12 +-
.../om/request/file/OMDirectoryCreateRequest.java | 27 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 6 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 22 +-
.../request/file/OMFileCreateRequestWithFSO.java | 7 +-
.../ozone/om/request/file/OMFileRequest.java | 140 +-
.../om/request/key/OMAllocateBlockRequest.java | 38 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 6 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 108 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 35 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 39 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 6 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 88 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 8 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 30 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 12 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 73 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 5 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 9 +-
.../om/request/key/OMPathsPurgeRequestWithFSO.java | 3 +-
.../om/request/key/OMTrashRecoverRequest.java | 2 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 58 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 10 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 5 +-
.../request/key/acl/OMKeyAddAclRequestWithFSO.java | 5 +-
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 4 +-
.../key/acl/OMKeyRemoveAclRequestWithFSO.java | 6 +-
.../om/request/key/acl/OMKeySetAclRequest.java | 4 +-
.../request/key/acl/OMKeySetAclRequestWithFSO.java | 5 +-
.../S3InitiateMultipartUploadRequest.java | 33 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 8 +-
.../multipart/S3MultipartUploadAbortRequest.java | 49 +-
.../S3MultipartUploadAbortRequestWithFSO.java | 6 +-
.../S3MultipartUploadCommitPartRequest.java | 38 +-
.../S3MultipartUploadCommitPartRequestWithFSO.java | 7 +-
.../S3MultipartUploadCompleteRequest.java | 62 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 6 +-
.../ozone/om/request/upgrade/OMPrepareRequest.java | 9 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 26 +
.../response/file/OMDirectoryCreateResponse.java | 6 +-
.../response/file/OMFileCreateResponseWithFSO.java | 5 +
.../response/key/AbstractOMKeyDeleteResponse.java | 11 +-
.../om/response/key/OMAllocateBlockResponse.java | 4 +-
.../key/OMAllocateBlockResponseWithFSO.java | 6 +
.../ozone/om/response/key/OMKeyCommitResponse.java | 31 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 21 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 11 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 5 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 9 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 25 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 18 +-
.../om/response/key/OMKeysDeleteResponse.java | 3 +-
.../om/response/key/OMKeysRenameResponse.java | 11 +-
.../om/response/key/OMOpenKeysDeleteRequest.java | 5 +-
.../om/response/key/OMOpenKeysDeleteResponse.java | 3 +-
.../response/key/OMPathsPurgeResponseWithFSO.java | 11 +-
.../om/response/key/OMTrashRecoverResponse.java | 10 +-
.../ozone/om/response/key/OmKeyResponse.java | 71 +
.../om/response/key/acl/OMKeyAclResponse.java | 4 +-
.../response/key/acl/OMKeyAclResponseWithFSO.java | 4 +-
.../S3InitiateMultipartUploadResponse.java | 4 +-
.../multipart/S3MultipartUploadAbortResponse.java | 4 +-
.../S3MultipartUploadAbortResponseWithFSO.java | 6 +
.../S3MultipartUploadCommitPartResponse.java | 4 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 6 +
.../S3MultipartUploadCompleteResponse.java | 38 +-
.../protocolPB/OMAdminProtocolServerSideImpl.java | 66 +
...OzoneManagerProtocolServerSideTranslatorPB.java | 63 +-
.../protocolPB/OzoneManagerRequestHandler.java | 39 +-
.../hadoop/ozone/protocolPB/RequestHandler.java | 4 +-
.../security/OzoneBlockTokenSecretManager.java | 5 -
.../hadoop/ozone/security/S3SecurityUtil.java | 82 ++
.../hadoop/ozone/om/TestBucketManagerImpl.java | 63 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 7 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 4 +-
.../ozone/om/request/TestOMRequestUtils.java | 139 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 1 -
.../bucket/TestOMBucketCreateRequestWithFSO.java | 8 +-
.../request/file/TestOMDirectoryCreateRequest.java | 53 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 94 +-
.../om/request/file/TestOMFileCreateRequest.java | 8 +-
.../file/TestOMFileCreateRequestWithFSO.java | 26 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 7 +-
.../key/TestOMAllocateBlockRequestWithFSO.java | 25 +-
.../ozone/om/request/key/TestOMKeyAclRequest.java | 40 +-
.../om/request/key/TestOMKeyAclRequestWithFSO.java | 29 +-
.../om/request/key/TestOMKeyCommitRequest.java | 131 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 19 +-
.../om/request/key/TestOMKeyCreateRequest.java | 70 +-
.../request/key/TestOMKeyCreateRequestWithFSO.java | 18 +-
.../om/request/key/TestOMKeyDeleteRequest.java | 26 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 26 +-
.../om/request/key/TestOMKeyRenameRequest.java | 8 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 10 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 4 +-
.../om/request/key/TestOMKeysRenameRequest.java | 20 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 10 +-
.../TestS3InitiateMultipartUploadRequest.java | 30 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 14 +-
.../s3/multipart/TestS3MultipartRequest.java | 21 +-
.../TestS3MultipartUploadAbortRequest.java | 5 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 7 +-
.../TestS3MultipartUploadCommitPartRequest.java | 17 +-
...tS3MultipartUploadCommitPartRequestWithFSO.java | 19 +-
.../TestS3MultipartUploadCompleteRequest.java | 19 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 28 +-
.../ozone/om/response/TestCleanupTableInfo.java | 2 +
.../file/TestOMDirectoryCreateResponse.java | 7 +-
.../file/TestOMFileCreateResponseWithFSO.java | 13 +-
.../response/key/TestOMAllocateBlockResponse.java | 12 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 14 +-
.../om/response/key/TestOMKeyCommitResponse.java | 46 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 18 +-
.../om/response/key/TestOMKeyCreateResponse.java | 12 +-
.../key/TestOMKeyCreateResponseWithFSO.java | 19 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 18 +-
.../key/TestOMKeyDeleteResponseWithFSO.java | 14 +-
.../om/response/key/TestOMKeyRenameResponse.java | 36 +-
.../ozone/om/response/key/TestOMKeyResponse.java | 8 +
.../om/response/key/TestOMKeysDeleteResponse.java | 9 +-
.../om/response/key/TestOMKeysRenameResponse.java | 16 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 18 +-
.../TestS3InitiateMultipartUploadResponse.java | 3 +-
...stS3InitiateMultipartUploadResponseWithFSO.java | 9 +-
.../s3/multipart/TestS3MultipartResponse.java | 15 +-
.../TestS3MultipartUploadAbortResponse.java | 11 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 15 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 100 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 21 +-
hadoop-ozone/ozonefs-common/pom.xml | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 18 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 39 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 4 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 67 +
hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 6 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
hadoop-ozone/pom.xml | 6 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
.../recon/schema/ContainerSchemaDefinition.java | 3 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 9 +-
.../codec/ContainerReplicaHistoryListCodec.java | 45 +-
.../ozone/recon/fsck/ContainerHealthStatus.java | 17 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 3 +
.../persistence/ContainerHealthSchemaManager.java | 12 +-
.../ozone/recon/persistence/ContainerHistory.java | 9 +-
.../ozone/recon/scm/ContainerReplicaHistory.java | 27 +-
.../recon/scm/ContainerReplicaHistoryList.java | 32 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 3 +
.../ozone/recon/scm/ReconContainerManager.java | 21 +-
.../ozone/recon/scm/ReconPipelineManager.java | 10 +-
.../scm/ReconStorageContainerManagerFacade.java | 5 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 4 +-
.../impl/ReconContainerMetadataManagerImpl.java | 4 +-
.../ozone/recon/spi/impl/ReconDBDefinition.java | 13 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 8 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 25 +-
.../ozone/recon/api/TestContainerEndpoint.java | 34 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +-
.../ozone/recon/api/TestNSSummaryEndpoint.java | 3 -
.../ozone/recon/fsck/TestContainerHealthTask.java | 16 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 25 +-
.../ozone/recon/scm/TestReconContainerManager.java | 13 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 5 +
.../impl/TestOzoneManagerServiceProviderImpl.java | 17 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 9 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 9 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 16 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 30 +-
hadoop-ozone/s3gateway/pom.xml | 4 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 103 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 +
.../ozone/s3/endpoint/CopyObjectResponse.java | 2 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 29 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 59 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 5 +
.../hadoop/ozone/s3/io/S3WrapperInputStream.java | 84 --
.../apache/hadoop/ozone/s3/io/package-info.java | 23 -
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 22 +-
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +-
.../s3/endpoint/TestAbortMultipartUpload.java | 2 +
.../s3/endpoint/TestInitiateMultipartUpload.java | 2 +
.../hadoop/ozone/s3/endpoint/TestListParts.java | 2 +
.../s3/endpoint/TestMultipartUploadComplete.java | 2 +
.../s3/endpoint/TestMultipartUploadWithCopy.java | 221 +++-
.../hadoop/ozone/s3/endpoint/TestObjectDelete.java | 2 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 2 +
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 2 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 2 +
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 2 +
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +
hadoop-ozone/tools/pom.xml | 4 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 2 +-
.../admin/nssummary/FileSizeDistSubCommand.java | 2 +-
.../ozone/admin/nssummary/NSSummaryAdmin.java | 36 +-
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 8 +-
.../admin/nssummary/QuotaUsageSubCommand.java | 2 +-
.../ozone/admin/nssummary/SummarySubCommand.java | 2 +-
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 5 +-
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 5 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 17 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 80 +-
.../apache/hadoop/ozone/debug/PrefixParser.java | 26 +-
.../ozone/debug/container/ContainerCommands.java | 16 +-
.../ozone/freon/ClosedContainerReplicator.java | 10 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 8 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 155 +--
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 54 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 908 +++++++++++++
.../containergenerator/GeneratorDatanode.java | 23 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 7 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 2 +-
.../ozone/genesis/BenchMarkOMKeyAllocation.java | 137 --
.../org/apache/hadoop/ozone/genesis/Genesis.java | 2 +-
.../hadoop/ozone/segmentparser/RatisLogParser.java | 4 +-
...MRatisLogParser.java => SCMRatisLogParser.java} | 15 +-
.../org/apache/hadoop/ozone/shell/Handler.java | 17 +
.../ozone/shell/bucket/CreateBucketHandler.java | 24 +-
.../ozone/shell/bucket/ListBucketHandler.java | 10 +-
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 18 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 8 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 18 +-
.../ozone/shell/volume/ListVolumeHandler.java | 9 +-
.../ozone/debug/TestDBDefinitionFactory.java | 7 +-
pom.xml | 79 +-
579 files changed, 17504 insertions(+), 7888 deletions(-)
create mode 100644 .github/ci.md
rename hadoop-hdds/{framework =>
common}/src/main/java/org/apache/hadoop/hdds/NodeDetails.java (100%)
copy
hadoop-hdds/{container-service/src/main/java/org/apache/hadoop/ozone/container/stream/StreamingSource.java
=>
common/src/main/java/org/apache/hadoop/hdds/client/ReplicatedReplicationConfig.java}
(62%)
copy
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/{ReplicateContainerCommandHandler.java
=> RefreshVolumeUsageCommandHandler.java} (51%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorMetrics.java
copy hadoop-hdds/docs/{archetypes/default.md =>
themes/ozonedoc/layouts/shortcodes/image.html} (86%)
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerV2.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceStrategy.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/{PipelineStateManagerV2Impl.java
=> PipelineStateManagerImpl.java} (88%)
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/StateManager.java
copy
hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate
=>
server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer}/package-info.java
(84%)
rename
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/{TestPipelineStateManager.java
=> TestPipelineStateManagerImpl.java} (63%)
delete mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMStoreImplWithOldPipelineIDKeyFormat.java
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/RpcClientTest.java
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/{OMInterServiceProtocol.java
=> OMAdminProtocol.java} (80%)
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMConfiguration.java
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/{helpers/OmDeleteKeys.java
=> protocol/S3Auth.java} (56%)
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/{OMInterServiceProtocolPB.java
=> OMAdminProtocolPB.java} (81%)
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/{OmTransport.java
=> OzoneManagerClientProtocol.java} (56%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithLinks.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStandardOutputUtil.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisRequest.java
rename
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{om/parser/TestOMRatisLogParser.java
=> parser/TestOzoneHARatisLogParser.java} (68%)
copy
hadoop-ozone/interface-client/src/main/proto/{OmInterServiceProtocol.proto =>
OMAdminProtocol.proto} (58%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneAclUtils.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerUtils.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMKeyRequestFactory.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OmKeyResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java
delete mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
delete mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java
delete mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOMKeyAllocation.java
copy
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/segmentparser/{OMRatisLogParser.java
=> SCMRatisLogParser.java} (75%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
