This is an automated email from the ASF dual-hosted git repository.
pifta pushed a change to branch HDDS-5447-httpfs
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 0f3e560 HDDS-5827 [HTTPFSGW] Remove non-server side related code from
Ozone. (#2750)
add 7e2cdf0 HDDS-5432. Enable downgrade testing after 1.1.0 release.
(#2484)
add bbd0907 HDDS-5678. Handle unsecure SCM HA converted to secure SCM HA.
(#2596)
add 0decca8 HDDS-5716. Fix create key failure error log print (#2614)
add 737297e HDDS-5719. Reduce number of mini-clusters needed for
decommission tests (#2617)
add 2d188cc HDDS-5726. Skip remove for already removed pipeline. (#2624)
add b64080b HDDS-5638. Fix docker-compose to make Recon come up. (#2563)
add e9cce6e HDDS-5670. ContainerBalancer should get OzoneConfiguration
from ContainerBalancerConfiguration. (#2577)
add ebbcf0a HDDS-5711. support -1 for running balancer infinitely (#2621)
add 6239270 HDDS-5724. Add RaftpeerId when getting scm roles (#2622)
add 2b6e1ac HDDS-5718. Refactor TestXceiverClientManager to reuse
mini-clusters (#2616)
add ee993d0 HDDS-5723. Increase time limit of Ozone acceptance tests.
(#2620)
add e392c1a HDDS-5699. Added Log to show why a container was marked
UNHEALTHY. (#2627)
add 753a5b7 HDDS-5715. Make XceiverServerRatis#raftGids a thread-safe
set. (#2613)
add 6569040 HDDS-5502. [OFS] URI parser throws URISyntaxException when
path contains space (#2500)
add bb37a55 HDDS-5735. Prometheus HTTP API Reference cannot be displayed
normally (#2630)
add 67be640 HDDS-5496. Missing directory name in `start_k8s_env` log
(#2545)
add 9aebf6e HDDS-5720. Reuse mini-clusters in TestOzoneFileInterfaces
(#2619)
add e97ea36 HDDS-5679. Use more defensive sizeRequired for replication
manager for container replication. (#2585)
add 2d57e9f HDDS-5708. Skip sending container close command to unhealthy
replica. (#2610)
add 873c0d2 HDDS-5696. Close grpcReplicationClient on exception. (#2595)
add 6543c67 HDDS-5729. support setting maxSizeEnteringTarget and
maxSizeLeavingSource in command line (#2626)
add ec5c51f HDDS-5335. Method not found: allocateBlock - when tracing is
enabled (#2330)
add c5a012b HDDS-5744. Remove unnecessary duplicate semicolon in
PipelineStateMap (#2641)
add ee19b95 HDDS-5748. Reuse mini-clusters in
TestOzoneFSWithObjectStoreCreate (#2646)
add da18617 HDDS-5717. Refactor TestOzoneManagerListVolumes to reuse
mini-ozone clusters (#2615)
add ed30752 HDDS-5749. Reuse mini-clusters in TestOzoneFsHAURLs (#2647)
add d716bf4 HDDS-5402 Support list node based on NodeOperationalState and
NodeState options in printTopology CLI (#2645)
add fab8a12 HDDS-5728. ContainerBalancer should use remaining space to
calculate utilization. (#2625)
add f2c20ce HDDS-5751. Use Mini Cluster Provider to speed up
TestHDDSUpgrade (#2650)
add 5c972c4 HDDS-5753. Split parts of misc suite (#2654)
add 0b9e23e HDDS-5607. remove container manager v1 code (#2525)
add 2d58933 HDDS-5758. Speed up TestKeyInputStream and
TestChunkInputStream by combining some tests (#2656)
add c68468c HDDS-5761. should not shutdown om when setting a bigger
bucket quota than volume quota (#2659)
add 79da6b7 HDDS-5766. Speed up some OM tests by skipping SCM safemode
check (#2662)
add b8afbbc HDDS-5768. Skip safemode check in
TestOzoneManagerRocksDBLogging (#2665)
add 79716f0 HDDS-5765. Test cluster provider possibly returns null (#2663)
add 2c2d2f3 HDDS-5767. Unit check may timeout (#2664)
add b508b25 HDDS-5771. Speed up TestDatanodeHddsVolumeFailureToleration
by reducing dead interval (#2668)
add fc61be5 HDDS-5691. Restrict Recon NSSummaryEndpoint and
ContainerEndpoint to admins. (#2638)
add 3116fa2 HDDS-5773. Avoid code duplication for mini cluster without
datanodes (#2669)
add c25b6d7 HDDS-5759. Bump aspectj version (#2658)
add 87b4971 HDDS-5770. Silent failures of k3s install are difficult to
debug (#2667)
add 605a734 HDDS-5697. SCMContainerPlacementRackAware should both
affinity nodes before fallback (#2597)
add e5c647e HDDS-5370. [FSO] Handle OMClientRequest based on the bucket
layout. (#2533)
add 181266b HDDS-5777. Provide an option to dump table scan data to file.
(#2675)
add 4b3502f HDDS-5792. Speed up TestNSSummaryAdmin by having zero
Datanodes (#2688)
add 2e3c46c HDDS-5782. DatanodeRatisGrpcConfig use wrong type for config
keys. (#2678)
add 041684f HDDS-5789. Enabling container tokens on a upgraded cluster
fails SCM to start up. (#2691)
add af7dc7e HDDS-5800. Bucket usedByte was incorrect when deleting
duplicate keys.
add 8cc6719 HDDS-5523. Fix multipart upload failure in s3 compatibility
tests (#2509)
add aa9eade HDDS-5757. balancer should stop when the cluster can not be
balanced any more (#2676)
add e2c3ac3 HDDS-5747. Reset default values in
ContainerBalancerConfiguration (#2657)
add 5c2ac71 HDDS-5788. Reduce run time for TestOzoneManagerHA tests
(#2689)
add af5b48e HDDS-5636. Use bucket layout specific DB table in
OmMetadataManager:getOpenKeyTable(). (#2561)
add 093f972 HDDS-5787. Provide capability in Ozone to enable ratis
pre-vote (#2680)
add 05eb172 HDDS-5762. remove piplelineStateManager V1 code (#2661)
add c4cbcad HDDS-5710. initialize sequenceIdToLastIdMap when
SequenceIdGenerator#StateManager reinitializes (#2611)
add 53a0009 HDDS-5793. Speed up TestBlockOutputStreamWithFailures by
combining test cases (#2690)
add 1da0bd8 HDDS-5609. Document CI workflow (#2607)
add c886a2f HDDS-5797. Support setting Datanode Reserved Space in
MiniOzoneCluster. (#2695)
add 7bdc1bb HDDS-5799. Speed up TestOzoneFileSystem by using shorter
trash interval (#2699)
add 1ea1d6c HDDS-5808. Update commons-io to 2.11.0 (#2705)
add 186cd6e HDDS-5817. Extract properties for some dependency versions
(#2710)
add 20e8429 HDDS-5815 FileEncryptionInfo is not necessary to be
referenced inside KeyOutputStream. (#2708)
add a59c501 HDDS-5830. Increase idea.max.intellisense.filesize (#2715)
add 8d885b4 HDDS-5557. Cleanup
OzoneBlockTokenSecretManager#validateToken. (#2711)
add ea81ef8 HDDS-5816 Rearrange code and refactor some logic into new
methods in prep for EC addition. (#2709)
add 8eaa9c5 HDDS-4440. Diagrams for old and new approach are mixed up
(#2555)
add c5a92a2 HDDS-5824. `ozone sh volume/bucket/key list` should print
valid JSON array (#2713)
add db97a32 HDDS-5733. Incorrect calculation of iteration related metrics
in ContainerBalancer (#2631)
add efb5263 HDDS-5806. remove BackgroundPipelineCreator v1 code (#2700)
add 57729b4 HDDS-5536. Add metrics for ReplicationSupervisor (#2493)
add 1ffcfe9 HDDS-5349. Avoid usage of locks in listStatus. (#2369)
add 3e05e71 HDDS-5373. [FSO] Define default bucket layout. (#2707)
add f902b13 HDDS-5785. containerCreationLock is a severe bottleneck when
there are many containers to create. (#2679)
add a62c61f HDDS-5823. Change the key permission to 600. (#2722)
add 2f49993 HDDS-5845. Add metrics for stateMachineDataCache cache hit
and miss. (#2726)
add 730a2e8 HDDS-5842. Parse and dump SCM ratis segment file to printable
text (#2731)
add e19fa38 HDDS-5814. Handle InterruptedException in Ozone (#2706)
add ae812c06 HDDS-5831. Remove empty TaskQueue in ContainerStateMachine.
(#2716)
add ed069bd HDDS-5854. Bump json-smart to 2.3.1 (#2736)
add 7376cfe HDDS-5855. Bump commons-compress to 1.21 (#2737)
add 0e5ad31 HDDS-5856. Bump Spring to 5.2.17 (#2738)
add d341c05 HDDS-5857. Bump aws-java-sdk to latest 1.11 (#2740)
add 24ddf0f HDDS-5659. Use bucket layout specific DB table in
OmMetadataManager.getKeyTable() - part1 (#2724)
add bf6da11 HDDS-5858. Fix resource leak due to Files.list (#2741)
add f2f6aa5 HDDS-5671. ContainerBalancer#stop should prevent the current
balancing thread from interrupting itself. (#2578)
add bc81dcf HDDS-5734. add a command to trigger datanode executing du
immediately (#2685)
add a1701e5 HDDS-5850. Update default value of
'ozone.scm.ha.ratis.segment.size' and 'preallocated.size' to improve SCM write
perf. (#2732)
add 7d79d73 HDDS-5812. Increase Ratis version to 2.2.0 (#2753)
add fbb8cf0 HDDS-5794. The misleading "No available thread in pool for
past * second" log message in DN StateContext (#2693)
add 996518a HDDS-5860. Shade Jackson for Ozone Filesystem (#2744)
add 42d3de3 HDDS-5725. Catch possible AlreadyExistsException for create
pipeline command. (#2623)
add 2a4fcde HDDS-5888. Handle InterruptedException in OzoneClientProducer
and ContainerBalancer (#2761)
add ba038e4 HDDS-5875. Do not cache tokens as part of Key Information.
(#2754)
add 2a4bc83 HDDS-5461. Move old objects to delete table on overwrite
(#2433)
add 6e5db67 HDDS-5871. Fix ContainerBalancerConfiguration annotation
string (#2762)
add d8d42b5 HDDS-5868. Change getBucketInfo annotation (#2749)
add 0828afc HDDS-5843. SCM terminates when adding container to a pipeline
during startup (#2745)
add c5aaad8 HDDS-3371. Cleanup of old write-path of key in OM (#2703)
add fc666fa HDDS-5893. Potential data loss on multipart upload commit
(#2763)
add a51822f HDDS-5882 Changes to protobuf messages for HDDS-5881 (#2774)
add f133303 HDDS-5707. List container supports replication factor filter.
(#2606)
add 990db75 HDDS-5835. Follow up task to use bucket layout specific DB
table in OmMetadataManager.getKeyTable() - part2 (#2751)
add 0906337 HDDS-5902 Ambiguous OM kerberos keytab settings (#2769)
add a5d6707 HDDS-5901. Delete on volume/bucket throws fatal error on ofs
(#2776)
add 26824ca HDDS-5884. OM Validate S3 Auth for write requests. (#2778)
add d018d00 HDDS-5805. remove containerStateManager V1 code (#2727)
add f153236 HDDS-5863. Error message having null fields on volume
creation (#2752)
add 5c76856 HDDS-5885. OM Auth Validate for read requests and handle non
ratis enabled code. (#2779)
add 5f0a588 HDDS-5908. MPU getKey can fail, if completeMPU result is
still in cache. (#2787)
add e427b71 HDDS-5883 Change S3G client to set S3 Auth per req (#2775)
add 73a081a HDDS-5864. Retry when DN connection issue during
getBlock/ReadChunk call during Ozone key Read (#2746)
add 83f8b04 HDDS-5736. Update navbar from Hadoop to Ozone on doc page
(#2792)
add 47cb26a HDDS-3983 Ozone RocksDB Iterator wrapper should not expose
key() and value() API. (#2402)
add 565972c Support more detailed error log when handleFlush in Ozone
client (#2795)
add 1cf43ac HDDS-5931. Broken link in dist/README (#2799)
add 9cea16d HDDS-5933. Ozone Filesystem shaded jars include unnecessary
dependencies (#2798)
add eb7136d HDDS-3369. Cleanup old write-path of volume in OM. (#2780)
add c088cd1 HDDS-5925. Fix SecuringOzoneHTTP doc (#2796)
add ac86851 HDDS-5943. Shutdown ResultHandlerExecutorService for
StorageVolumeChecker. (#2806)
add aa7e083 HDDS-5944. Fix typo in OzoneManagerServiceProviderImpl.java
(#2807)
add 6ebb316 HDDS-5945. bump rocksdb version to 6.25.3 (#2809)
add 4c2181f HDDS-5935. Bump Spring to 5.2.18 (#2801)
add f3b482a HDDS-5922. ignore deletetransaction when container is not
found (#2793)
add c63479c HDDS-5839. Make sure buckets created from OFS are in
FILE_SYSTEM_OPTIMIZED layout (#2730)
add 96e2d7f HDDS-5947. Remove unused mina-core and sshd-core dependencies
(#2811)
add 252fee4 HDDS-5910 Add additional verification for S3 Auth. (#2817)
add 9d9e780 HDDS-5956. Speed up
TestOzoneRpcClientAbstract#testZReadKeyWithUnhealthyContainerReplica (#2820)
add 0eedab9 HDDS-5929. Make FSO and OBS bucket layouts independent of
normalization config flag (#2819)
add ba3ac30 HDDS-5960. Change option name to bucketlayout instead of type
(#2825)
add 4558e40 HDDS-5970. Remove OMKeyRequest#getBucketLayout overridden
method in subclasses (#2830)
add 9384bd7 HDDS-5820. Intermittent failure in
TestPipelinePlacementPolicy#testPickLowestLoadAnchor (#2757)
add d0e9140 HDDS-5958. Refine container scrub log message. (#2823)
add 45c1899 HDDS-5937. Inaccurate bucket info returned from bucket list
command in Shell (#2832)
add c2db377 HDDS-5872. Do not failover on some RpcExceptions (#2772)
add 71f47d4 HDDS-5981. Refactor usage of bucket type. (#2836)
add 026d878 HDDS-5982. Remove isBucketFSOptimized flag. (#2838)
add 346b55d HDDS-5906. Fix DBScanner to support Datanode DB (#2781)
add 1b7072c HDDS-5983. Prefix Parser tool should only work for FSO
buckets. (#2839)
add 5ef59b4 HDDS-5957. Bump proto-backwards-compatibility from 1.0.5 to
1.0.7 (#2821)
add b4a785c HDDS-5891. OFS mkdir -p does not work as expected for bucket
creation when volume exists due to volume create ACL check (#2814)
new 56f104c Merge branch 'master' into HDDS-5447-httpfs
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/ci.md | 102 +++
.github/workflows/post-commit.yml | 6 +-
CONTRIBUTING.md | 4 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 52 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 +
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 14 +-
.../hdds/scm/storage/TestBlockInputStream.java | 32 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 51 ++
.../java/org/apache/hadoop/hdds/NodeDetails.java | 0
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 23 +-
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 10 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 13 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 11 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 4 +
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 31 +-
.../hadoop/hdds/scm/net/NetworkTopology.java | 3 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 5 +-
.../protocol/StorageContainerLocationProtocol.java | 24 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 5 -
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 23 +-
.../apache/hadoop/hdds/tracing/TracingUtil.java | 6 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 -
.../upgrade/AbstractLayoutVersionManager.java | 158 ++--
.../common/src/main/resources/ozone-default.xml | 84 +-
.../hadoop/hdds/tracing/TestTraceAllMethod.java | 68 ++
.../hadoop/hdds/tracing/TestTracingUtil.java} | 33 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 5 +-
.../ozone/container/common/impl/ContainerSet.java | 3 +
.../container/common/impl/HddsDispatcher.java | 1 +
.../common/interfaces/ContainerDispatcher.java | 7 +-
.../common/statemachine/DatanodeStateMachine.java | 19 +-
.../common/statemachine/StateContext.java | 14 +-
.../CloseContainerCommandHandler.java | 2 +
.../ClosePipelineCommandHandler.java | 13 +-
.../CreatePipelineCommandHandler.java | 10 +-
....java => RefreshVolumeUsageCommandHandler.java} | 67 +-
.../states/endpoint/VersionEndpointTask.java | 5 +-
.../common/transport/server/ratis/CSMMetrics.java | 10 +
.../server/ratis/ContainerStateMachine.java | 9 +-
.../transport/server/ratis/XceiverServerRatis.java | 15 +-
.../container/common/utils/HddsVolumeUtil.java | 72 +-
.../ozone/container/common/volume/HddsVolume.java | 2 +-
.../container/common/volume/MutableVolumeSet.java | 17 +-
.../container/common/volume/StorageVolume.java | 4 +
.../common/volume/StorageVolumeChecker.java | 1 +
.../ozone/container/common/volume/VolumeInfo.java | 13 +-
.../ozone/container/common/volume/VolumeUsage.java | 3 +
.../container/keyvalue/KeyValueContainer.java | 12 +-
.../container/keyvalue/KeyValueContainerCheck.java | 4 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 34 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 7 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 6 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 161 +---
.../ozone/container/ozoneimpl/OzoneContainer.java | 5 +-
.../replication/DownloadAndImportReplicator.java | 5 +-
.../replication/ReplicationSupervisor.java | 8 +
.../replication/ReplicationSupervisorMetrics.java | 72 ++
.../replication/SimpleContainerDownloader.java | 27 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 5 +-
.../upgrade/DatanodeMetadataFeatures.java | 53 --
.../ScmHAFinalizeUpgradeActionDatanode.java | 129 +++
.../ScmHAFirstUpgradeLayoutChangeAction.java | 45 --
.../upgrade/VersionedDatanodeFeatures.java | 145 ++++
.../hadoop/ozone/container/common/ScmTestMock.java | 9 +
.../ozone/container/common/TestContainerCache.java | 23 +-
.../common/TestKeyValueContainerData.java | 8 +-
.../common/impl/TestContainerDataYaml.java | 6 +-
.../common/volume/TestStorageVolumeChecker.java | 4 +
.../replication/TestReplicationSupervisor.java | 59 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 741 +++++++++++++++++
hadoop-hdds/docs/content/feature/PrefixFSO.md | 16 +
hadoop-hdds/docs/content/feature/Topology.md | 2 +-
hadoop-hdds/docs/content/interface/CSI.md | 4 +-
hadoop-hdds/docs/content/interface/CSI.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Cli.md | 7 +-
hadoop-hdds/docs/content/interface/ReconApi.md | 23 +-
hadoop-hdds/docs/content/interface/ReconApi.zh.md | 2 +-
.../docs/content/security/SecuringOzoneHTTP.md | 20 +-
hadoop-hdds/docs/content/start/FromSource.md | 2 +-
.../docs/static/design/s3-performance-new.png | Bin 33688 -> 34927 bytes
.../docs/static/design/s3-performance-old.png | Bin 34927 -> 33688 bytes
.../themes/ozonedoc/layouts/partials/navbar.html | 6 +-
.../themes/ozonedoc/layouts/partials/sidebar.html | 4 +-
.../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java | 4 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 16 +
...inerLocationProtocolClientSideTranslatorPB.java | 31 +-
.../certificate/client/DNCertificateClient.java | 5 -
.../client/DefaultCertificateClient.java | 2 +-
.../certificate/client/OMCertificateClient.java | 5 -
.../certificate/client/SCMCertificateClient.java | 10 +-
.../hadoop/hdds/security/x509/keys/KeyCodec.java | 31 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 16 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 19 +-
.../apache/hadoop/hdds/utils/db/TableIterator.java | 12 -
.../apache/hadoop/hdds/utils/db/TypedTable.java | 18 -
.../hdds/security/x509/keys/TestKeyCodec.java | 5 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 19 +-
.../src/main/proto/ScmAdminProtocol.proto | 3 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 6 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 22 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 9 +-
.../container/AbstractContainerReportHandler.java | 6 +-
.../scm/container/CloseContainerEventHandler.java | 4 +-
.../hdds/scm/container/ContainerManager.java | 144 ++--
.../hdds/scm/container/ContainerManagerImpl.java | 8 +-
.../hdds/scm/container/ContainerManagerV2.java | 190 -----
.../hdds/scm/container/ContainerReportHandler.java | 6 +-
.../hdds/scm/container/ContainerStateManager.java | 570 +++----------
.../scm/container/ContainerStateManagerImpl.java | 19 +-
.../scm/container/ContainerStateManagerV2.java | 189 -----
.../IncrementalContainerReportHandler.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 34 +-
.../hdds/scm/container/SCMContainerManager.java | 672 ----------------
.../scm/container/balancer/ContainerBalancer.java | 287 ++++---
.../balancer/ContainerBalancerConfiguration.java | 58 +-
.../balancer/ContainerBalancerMetrics.java | 115 +--
.../ContainerBalancerSelectionCriteria.java | 14 +-
.../scm/container/balancer/FindTargetGreedy.java | 6 +-
.../algorithms/SCMContainerPlacementRackAware.java | 72 +-
.../container/placement/metrics/SCMNodeStat.java | 23 -
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 11 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 24 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 26 +-
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 65 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 6 +-
.../hdds/scm/node/NodeDecommissionManager.java | 8 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 8 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 60 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 264 ++++--
.../scm/pipeline/BackgroundPipelineCreatorV2.java | 332 --------
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 2 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 10 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 29 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 6 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 271 ++-----
...erV2Impl.java => PipelineStateManagerImpl.java} | 70 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 29 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 20 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
.../hadoop/hdds/scm/pipeline/StateManager.java | 127 ---
.../pipeline/WritableRatisContainerProvider.java | 6 +-
.../algorithms/DefaultLeaderChoosePolicy.java | 4 +-
.../choose/algorithms/LeaderChoosePolicy.java | 8 +-
.../algorithms/LeaderChoosePolicyFactory.java | 8 +-
.../algorithms/MinLeaderCountChoosePolicy.java | 6 +-
...inerLocationProtocolServerSideTranslatorPB.java | 21 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 10 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 10 +-
.../scm/server/OzoneStorageContainerManager.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 70 +-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 8 +-
.../hdds/scm/server/StorageContainerManager.java | 73 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 30 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 6 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 16 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 5 +
.../container/TestCloseContainerEventHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 8 +-
.../scm/container/TestContainerReportHandler.java | 269 ++++---
.../scm/container/TestContainerStateManager.java | 91 ++-
.../TestIncrementalContainerReportHandler.java | 130 +--
.../hdds/scm/container/TestReplicationManager.java | 395 +++++----
.../scm/container/TestSCMContainerManager.java | 324 --------
.../scm/container/TestUnknownContainerReport.java | 51 +-
.../container/balancer/TestContainerBalancer.java | 91 ++-
.../hdds/scm/container/balancer}/package-info.java | 7 +-
.../TestSCMContainerPlacementRackAware.java | 15 +
.../hdds/scm/ha/TestReplicationAnnotation.java | 8 +-
.../hdds/scm/node/TestContainerPlacement.java | 122 +--
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 8 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 33 +-
.../scm/pipeline/MockRatisPipelineProvider.java | 6 +-
.../TestPipelineDatanodesIntersection.java | 52 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 26 +
.../scm/pipeline/TestPipelinePlacementPolicy.java | 73 +-
...ager.java => TestPipelineStateManagerImpl.java} | 237 ++++--
.../scm/pipeline/TestRatisPipelineProvider.java | 66 +-
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 209 -----
.../scm/pipeline/TestSimplePipelineProvider.java | 48 +-
.../choose/algorithms/TestLeaderChoosePolicy.java | 6 +-
.../ozone/container/common/TestEndPoint.java | 4 +-
.../testutils/ReplicationNodeManagerMock.java | 11 +
.../scm/cli/ContainerBalancerStartSubcommand.java | 13 +-
.../hdds/scm/cli/ContainerOperationClient.java | 12 +-
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 46 +-
.../hdds/scm/cli/container/ListSubcommand.java | 6 +-
.../datanode/TestContainerBalancerSubCommand.java | 6 +-
.../org/apache/hadoop/ozone/client/BucketArgs.java | 8 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 268 +++++--
.../client/io/BlockOutputStreamEntryPool.java | 109 ++-
.../hadoop/ozone/client/io/KeyOutputStream.java | 16 +-
.../ozone/client/protocol/ClientProtocol.java | 19 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 55 +-
hadoop-ozone/common/pom.xml | 4 -
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 48 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 27 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 18 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 11 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 25 +
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 17 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 7 -
.../ozone/om/protocol/OzoneManagerProtocol.java | 168 +++-
.../apache/hadoop/ozone/om/protocol/S3Auth.java} | 41 +-
...nsport.java => OzoneManagerClientProtocol.java} | 33 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 59 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 41 +-
.../org/apache/hadoop/ozone/csi/NodeService.java | 10 +-
hadoop-ozone/dev-support/checks/_lib.sh | 8 +-
hadoop-ozone/dist/README.md | 6 +-
.../dist/src/main/compose/ozone-ha/test.sh | 2 +
.../dist/src/main/compose/ozone-mr/common-config | 1 +
.../src/main/compose/ozone-mr/hadoop27/test.sh | 2 +
.../src/main/compose/ozone-mr/hadoop31/test.sh | 2 +
.../src/main/compose/ozone-mr/hadoop32/test.sh | 2 +
.../src/main/compose/ozone-mr/hadoop33/test.sh | 2 +
.../dist/src/main/compose/ozone-mr/test.sh | 3 +
.../compose/ozonesecure-ha/docker-compose.yaml | 2 +
.../dist/src/main/compose/ozonesecure-ha/test.sh | 2 +
.../dist/src/main/compose/ozonesecure-mr/test.sh | 2 +
.../src/main/compose/ozonesecure/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 16 -
.../dist/src/main/compose/upgrade/README.md | 35 +-
.../compose/upgrade/delete-and-regenerate-data.sh | 53 --
.../dist/src/main/compose/upgrade/delete-data.sh | 36 -
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 9 +-
.../manual-upgrade/0.5.0-1.1.0/callback.sh | 40 -
.../upgrades/manual-upgrade}/README.md | 9 +-
.../{1.0.0-1.1.0 => 1.1.0-1.2.0}/callback.sh | 17 +-
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 31 +-
.../dist/src/main/compose/versions/0.5.0.sh | 26 -
.../dist/src/main/compose/versions/1.0.0.sh | 26 -
.../dist/src/main/compose/versions/1.1.0.sh | 26 -
.../dist/src/main/compose/versions/README.md | 20 -
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 4 -
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 2 +-
.../src/main/smoketest/admincli/container.robot | 8 +
.../dist/src/main/smoketest/basic/links.robot | 8 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 10 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 2 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 18 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 53 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 23 +-
.../main/smoketest/security/ozone-secure-fs.robot | 2 +-
.../dist/src/main/smoketest/topology/cli.robot | 6 +
.../dev-support/findbugsExcludeFile.xml | 4 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 19 +-
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 33 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 107 ++-
.../fs/ozone/TestOzoneFileInterfacesWithFSO.java | 29 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 44 +-
.../fs/ozone/TestOzoneFileSystemMetrics.java | 3 +
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 9 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 9 +-
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 35 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 102 ++-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 7 -
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 4 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 4 +-
.../TestContainerStateManagerIntegration.java | 138 ++--
.../metrics/TestSCMContainerManagerMetrics.java | 4 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 4 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 4 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 41 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 31 +
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 7 +
.../hadoop/ozone/MiniOzoneClusterProvider.java | 3 +
.../ozone/TestContainerBalancerOperations.java | 8 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 16 +-
.../rpc/TestBlockOutputStreamWithFailures.java | 20 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 20 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 27 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 135 +++-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 100 ++-
.../client/rpc/read/TestChunkInputStream.java | 13 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 44 -
.../ozone/client/rpc/read/TestKeyInputStream.java | 58 +-
.../apache/hadoop/ozone/container/TestHelper.java | 4 +-
.../commandhandler/TestBlockDeletion.java | 10 +-
.../container/ozoneimpl/TestOzoneContainer.java | 8 +-
.../ozone/dn/TestDatanodeLayoutUpgradeTool.java | 131 ---
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 4 +-
.../TestDatanodeHddsVolumeFailureToleration.java | 40 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 44 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 10 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 103 +++
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 26 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 27 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 64 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 301 ++++---
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 114 ++-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 98 +--
.../ozone/om/TestOzoneManagerHAWithFailover.java | 65 ++
.../ozone/om/TestOzoneManagerListVolumes.java | 117 +--
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 28 +-
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 7 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 71 ++
.../TestOzoneHARatisLogParser.java} | 52 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 10 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 4 +-
.../ozone/recon/TestReconWithOzoneManager.java | 19 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 6 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 4 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 18 +-
.../scm/node/TestDecommissionAndMaintenance.java | 94 +--
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 9 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 142 +++-
.../src/main/proto/OmClientProtocol.proto | 11 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 6 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 5 +-
hadoop-ozone/ozone-manager/pom.xml | 10 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 21 +-
.../hadoop/ozone/om/DirectoryDeletingService.java | 21 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 777 +++++++++++-------
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 14 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 29 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 893 +++------------------
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 7 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 1 +
.../org/apache/hadoop/ozone/om/VolumeManager.java | 40 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 300 -------
.../ozone/om/ratis/OzoneManagerRatisServer.java | 18 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 206 +++--
.../hadoop/ozone/om/request/OMClientRequest.java | 22 +-
.../om/request/bucket/OMBucketCreateRequest.java | 11 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 9 +-
.../om/request/file/OMDirectoryCreateRequest.java | 10 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 6 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 9 +-
.../request/file/OMFileCreateRequestWithFSO.java | 9 +-
.../ozone/om/request/file/OMFileRequest.java | 140 +++-
.../om/request/key/OMAllocateBlockRequest.java | 24 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 6 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 95 ++-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 35 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 29 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 6 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 74 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 8 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 14 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 9 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 71 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 5 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 9 +-
.../om/request/key/OMPathsPurgeRequestWithFSO.java | 6 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 47 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 5 +-
.../S3InitiateMultipartUploadRequest.java | 17 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 6 +-
.../multipart/S3MultipartUploadAbortRequest.java | 34 +-
.../S3MultipartUploadAbortRequestWithFSO.java | 6 +-
.../S3MultipartUploadCommitPartRequest.java | 23 +-
.../S3MultipartUploadCommitPartRequestWithFSO.java | 7 +-
.../S3MultipartUploadCompleteRequest.java | 35 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 6 +-
.../ozone/om/request/upgrade/OMPrepareRequest.java | 9 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 5 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 26 +
.../response/file/OMDirectoryCreateResponse.java | 6 +-
.../response/file/OMFileCreateResponseWithFSO.java | 5 +
.../om/response/key/OMAllocateBlockResponse.java | 4 +-
.../key/OMAllocateBlockResponseWithFSO.java | 6 +
.../ozone/om/response/key/OMKeyCommitResponse.java | 26 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 21 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 6 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 3 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 9 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 12 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 14 +-
.../om/response/key/OMKeysDeleteResponse.java | 3 +-
.../om/response/key/OMKeysRenameResponse.java | 11 +-
.../om/response/key/OMOpenKeysDeleteRequest.java | 5 +-
.../om/response/key/OMOpenKeysDeleteResponse.java | 3 +-
.../response/key/OMPathsPurgeResponseWithFSO.java | 9 +-
.../om/response/key/OMTrashRecoverResponse.java | 2 +-
.../om/response/key/acl/OMKeyAclResponse.java | 4 +-
.../response/key/acl/OMKeyAclResponseWithFSO.java | 4 +-
.../S3InitiateMultipartUploadResponse.java | 4 +-
.../multipart/S3MultipartUploadAbortResponse.java | 4 +-
.../S3MultipartUploadAbortResponseWithFSO.java | 6 +
.../S3MultipartUploadCommitPartResponse.java | 4 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 6 +
.../S3MultipartUploadCompleteResponse.java | 38 +-
.../ozone/om/upgrade/OMLayoutVersionManager.java | 3 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 57 +-
.../protocolPB/OzoneManagerRequestHandler.java | 37 +-
.../security/OzoneBlockTokenSecretManager.java | 5 -
.../hadoop/ozone/security/S3SecurityUtil.java | 82 ++
.../hadoop/ozone/om/TestBucketManagerImpl.java | 63 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 7 +-
.../ozone/om/request/TestOMRequestUtils.java | 125 ++-
.../bucket/TestOMBucketCreateRequestWithFSO.java | 5 +
.../bucket/TestOMBucketSetPropertyRequest.java | 30 +-
.../request/file/TestOMDirectoryCreateRequest.java | 53 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 91 ++-
.../om/request/file/TestOMFileCreateRequest.java | 8 +-
.../file/TestOMFileCreateRequestWithFSO.java | 26 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 7 +-
.../key/TestOMAllocateBlockRequestWithFSO.java | 25 +-
.../ozone/om/request/key/TestOMKeyAclRequest.java | 30 +-
.../om/request/key/TestOMKeyAclRequestWithFSO.java | 15 +-
.../om/request/key/TestOMKeyCommitRequest.java | 131 ++-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 19 +-
.../om/request/key/TestOMKeyCreateRequest.java | 70 +-
.../request/key/TestOMKeyCreateRequestWithFSO.java | 20 +-
.../om/request/key/TestOMKeyDeleteRequest.java | 26 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 26 +-
.../om/request/key/TestOMKeyRenameRequest.java | 8 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 17 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 4 +-
.../om/request/key/TestOMKeysRenameRequest.java | 20 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 10 +-
.../TestS3InitiateMultipartUploadRequest.java | 30 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 14 +-
.../s3/multipart/TestS3MultipartRequest.java | 21 +-
.../TestS3MultipartUploadAbortRequest.java | 5 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 7 +-
.../TestS3MultipartUploadCommitPartRequest.java | 17 +-
...tS3MultipartUploadCommitPartRequestWithFSO.java | 19 +-
.../TestS3MultipartUploadCompleteRequest.java | 19 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 28 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 28 +-
.../file/TestOMDirectoryCreateResponse.java | 7 +-
.../file/TestOMFileCreateResponseWithFSO.java | 13 +-
.../response/key/TestOMAllocateBlockResponse.java | 12 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 14 +-
.../om/response/key/TestOMKeyCommitResponse.java | 46 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 18 +-
.../om/response/key/TestOMKeyCreateResponse.java | 12 +-
.../key/TestOMKeyCreateResponseWithFSO.java | 19 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 18 +-
.../key/TestOMKeyDeleteResponseWithFSO.java | 14 +-
.../om/response/key/TestOMKeyRenameResponse.java | 30 +-
.../ozone/om/response/key/TestOMKeyResponse.java | 8 +
.../om/response/key/TestOMKeysDeleteResponse.java | 9 +-
.../om/response/key/TestOMKeysRenameResponse.java | 16 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 18 +-
.../TestS3InitiateMultipartUploadResponse.java | 3 +-
...stS3InitiateMultipartUploadResponseWithFSO.java | 9 +-
.../s3/multipart/TestS3MultipartResponse.java | 15 +-
.../TestS3MultipartUploadAbortResponse.java | 11 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 15 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 100 ++-
.../hadoop/ozone/security/acl/TestParentAcl.java | 21 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 3 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 25 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 4 +-
.../org/apache/hadoop/fs/ozone/TestOFSPath.java | 26 +
hadoop-ozone/ozonefs-shaded/pom.xml | 2 +
hadoop-ozone/pom.xml | 2 +-
.../apache/hadoop/ozone/recon/ReconHttpServer.java | 1 +
.../hadoop/ozone/recon/ReconRestServletModule.java | 144 ++--
.../org/apache/hadoop/ozone/recon/ReconServer.java | 26 +-
.../apache/hadoop/ozone/recon/api/AdminOnly.java | 23 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 10 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 5 +-
.../ozone/recon/api/filters/ReconAdminFilter.java | 113 +++
.../ozone/recon/api/filters/ReconAuthFilter.java | 117 +++
.../recon/api/{ => filters}/package-info.java | 5 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 9 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 3 +
.../recon/scm/ReconContainerReportHandler.java | 4 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 4 +-
.../ReconIncrementalContainerReportHandler.java | 4 +-
.../ozone/recon/scm/ReconPipelineManager.java | 10 +-
.../scm/ReconStorageContainerManagerFacade.java | 4 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 4 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 8 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 25 +-
.../ozone/recon/api/TestContainerEndpoint.java | 8 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +-
.../ozone/recon/api/TestNSSummaryEndpoint.java | 15 +-
.../ozone/recon/api/filters/TestAdminFilter.java | 161 ++++
.../ozone/recon/fsck/TestContainerHealthTask.java | 6 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 25 +-
.../impl/TestOzoneManagerServiceProviderImpl.java | 17 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 9 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 9 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 16 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 30 +-
.../hadoop/ozone/s3/OzoneClientProducer.java | 77 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 +
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 29 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 40 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 5 +
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +-
.../s3/endpoint/TestMultipartUploadWithCopy.java | 219 +++--
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 5 +-
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 5 +-
.../admin/scm/GetScmRatisRolesSubcommand.java | 7 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 17 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 80 +-
.../apache/hadoop/ozone/debug/DatanodeLayout.java | 110 ---
.../apache/hadoop/ozone/debug/PrefixParser.java | 16 +-
.../ozone/debug/container/ContainerCommands.java | 16 +-
.../ozone/freon/ClosedContainerReplicator.java | 10 +-
.../containergenerator/GeneratorDatanode.java | 23 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 7 +-
.../ozone/genesis/BenchMarkOMKeyAllocation.java | 137 ----
.../org/apache/hadoop/ozone/genesis/Genesis.java | 2 +-
.../hadoop/ozone/segmentparser/RatisLogParser.java | 4 +-
...MRatisLogParser.java => SCMRatisLogParser.java} | 15 +-
.../org/apache/hadoop/ozone/shell/Handler.java | 17 +
.../ozone/shell/bucket/CreateBucketHandler.java | 10 +-
.../ozone/shell/bucket/ListBucketHandler.java | 10 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 8 +-
.../ozone/shell/volume/ListVolumeHandler.java | 9 +-
.../ozone/debug/TestDBDefinitionFactory.java | 7 +-
pom.xml | 74 +-
520 files changed, 11426 insertions(+), 9158 deletions(-)
create mode 100644 .github/ci.md
rename hadoop-hdds/{framework =>
common}/src/main/java/org/apache/hadoop/hdds/NodeDetails.java (100%)
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTraceAllMethod.java
copy
hadoop-hdds/{framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java
=> common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java}
(50%)
copy
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/{ReplicateContainerCommandHandler.java
=> RefreshVolumeUsageCommandHandler.java} (51%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorMetrics.java
delete mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeMetadataFeatures.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java
delete mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFirstUpgradeLayoutChangeAction.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/VersionedDatanodeFeatures.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToScmHA.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerV2.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerV2.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/{PipelineStateManagerV2Impl.java
=> PipelineStateManagerImpl.java} (88%)
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/StateManager.java
delete mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
copy
hadoop-hdds/{framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate
=>
server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer}/package-info.java
(84%)
rename
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/{TestPipelineStateManager.java
=> TestPipelineStateManagerImpl.java} (64%)
delete mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMStoreImplWithOldPipelineIDKeyFormat.java
copy
hadoop-ozone/{httpfsgateway/src/main/java/org/apache/ozone/lib/wsrs/ShortParam.java
=> common/src/main/java/org/apache/hadoop/ozone/om/protocol/S3Auth.java} (56%)
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/{OmTransport.java
=> OzoneManagerClientProtocol.java} (56%)
delete mode 100755
hadoop-ozone/dist/src/main/compose/upgrade/delete-and-regenerate-data.sh
delete mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/delete-data.sh
delete mode 100755
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/manual-upgrade/0.5.0-1.1.0/callback.sh
copy hadoop-ozone/dist/src/main/compose/{ozonesecure =>
upgrade/upgrades/manual-upgrade}/README.md (66%)
rename
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/{1.0.0-1.1.0
=> 1.1.0-1.2.0}/callback.sh (86%)
delete mode 100644 hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh
delete mode 100644 hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh
delete mode 100644 hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh
delete mode 100644 hadoop-ozone/dist/src/main/compose/versions/README.md
delete mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeLayoutUpgradeTool.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisRequest.java
rename
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{om/parser/TestOMRatisLogParser.java
=> parser/TestOzoneHARatisLogParser.java} (68%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java
copy
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditAction.java
=>
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/AdminOnly.java
(61%)
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/filters/ReconAdminFilter.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/filters/ReconAuthFilter.java
copy hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/{ =>
filters}/package-info.java (87%)
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/filters/TestAdminFilter.java
delete mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DatanodeLayout.java
delete mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOMKeyAllocation.java
copy
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/segmentparser/{OMRatisLogParser.java
=> SCMRatisLogParser.java} (75%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
