This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a change to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from c9442d2 HDDS-5778. Client side unit tests (#2810)
add 52ec27c HDDS-5610. Remove or clarify doc regarding OM federation
(#2529)
add 541a0dc HDDS-5640. Add scm ratis metrics (#2553)
add d90bc30 HDDS-5495. Ozone version mismatch in Kubernetes test lib
(#2544)
add 33fcd77 HDDS-5556. GrpcReplication Client might fail in SCM HA
Cluster (#2556)
add 1d8bfe0 HDDS-5540. Acceptance test: able to generate result in
xunit/junit format as well (#2514)
add d3f223c HDDS-5658. Handle InterruptedException in
DeleteBlocksCommandHandler and ContainerBalancer (#2560)
add b1eaf8f HDDS-5393. Return latest version of key location for client
on createKey/createFile (#2432)
add 6ce3036 HDDS-5554. Option to disable checksum verification. (#2522)
add f92ee75 HDDS-5619. Ozone data corruption issue on follower node.
(#2538)
add 03b0b5b HDDS-5391 Fix issues in SCMHAConfiguration (#2567)
add 318456d HDDS-5637. Add documentation for Recon Namespace REST APIs
(#2566)
add 52e9bb5 HDDS-5655. SCM terminates when allocatecontainer happens on
closed pipeline. (#2569)
add fdbf8c3 HDDS-5668. Fix TestOzoneConfigurationFields. (#2576)
add af6c018 HDDS-5637. Addendum: Add documentation for Recon Namespace
REST APIs (#2579)
add 5b4b21a HDDS-5521. Use short principal name as owner of a volume.
(#2573)
add 25561f8 HDDS-5623. Add data validation checks on datanode. (#2542)
add e12a6b9 HDDS-5611. Fixed NullPointerException in
ContainerStateMachine during Pipeline Close. (#2548)
add 30859fe HDDS-5687. support Optional<T> as parameters of commandLine
(#2588)
add 7dcaad1 HDDS-5681. Speed up TestOzoneManagerHAWithACL (#2587)
add b593fa8 HDDS-5688. Rpc should not retry if exception is
ContainerNotFoundException. (#2589)
add db75046 HDDS-5662. Fix blank traceId for FsShell command (#2568)
add 2958ab3 HDDS-5535. SCM-HA documents configuration error (#2492)
add 33b3f9e HDDS-5669. Fix NSSummaryEndpoint's NPE bug during OM
initialization (#2592)
add aab19a6 HDDS-5368. Add CLI command: ozone admin namespace summary
(#2565)
add 9baca7c HDDS-5692. Combine some tests in TestOzoneManagerHAWithData
to reuse mini-Clusters (#2593)
add 758b8ca HDDS-5689. chooseRandom in NetworkTopology should exclude the
affinity node too (#2590)
add 194de5b HDDS-5680. Fix bug for incomplete SCM ratis log (#2586)
add cdac8bc HDDS-5644. Speed up decommission tests using a background
Mini Cluster provider (#2554)
add fe42640 HDDS-5667. documentation page layout (#2604)
add 730cb66 HDDS-5706. Fix ReplicationManager zero metrics for inflight
actions. (#2605)
add ea1ff1e HDDS-5690. Speed up TestContainerReplication by removing
testSkipDemmissionAndMaintenanceNode (#2591)
add f71474a HDDS-5700. Improve LOG message of decommission progress.
(#2598)
add aa8cfcd HDDS-5709. do not call removeTransactionsFromDB if nothing to
remove (#2608)
add 7e2cdf0 HDDS-5432. Enable downgrade testing after 1.1.0 release.
(#2484)
add bbd0907 HDDS-5678. Handle unsecure SCM HA converted to secure SCM HA.
(#2596)
add 0decca8 HDDS-5716. Fix create key failure error log print (#2614)
add 737297e HDDS-5719. Reduce number of mini-clusters needed for
decommission tests (#2617)
add 2d188cc HDDS-5726. Skip remove for already removed pipeline. (#2624)
add b64080b HDDS-5638. Fix docker-compose to make Recon come up. (#2563)
add e9cce6e HDDS-5670. ContainerBalancer should get OzoneConfiguration
from ContainerBalancerConfiguration. (#2577)
add ebbcf0a HDDS-5711. support -1 for running balancer infinitely (#2621)
add 6239270 HDDS-5724. Add RaftpeerId when getting scm roles (#2622)
add 2b6e1ac HDDS-5718. Refactor TestXceiverClientManager to reuse
mini-clusters (#2616)
add ee993d0 HDDS-5723. Increase time limit of Ozone acceptance tests.
(#2620)
add e392c1a HDDS-5699. Added Log to show why a container was marked
UNHEALTHY. (#2627)
add 753a5b7 HDDS-5715. Make XceiverServerRatis#raftGids a thread-safe
set. (#2613)
add 6569040 HDDS-5502. [OFS] URI parser throws URISyntaxException when
path contains space (#2500)
add bb37a55 HDDS-5735. Prometheus HTTP API Reference cannot be displayed
normally (#2630)
add 67be640 HDDS-5496. Missing directory name in `start_k8s_env` log
(#2545)
add 9aebf6e HDDS-5720. Reuse mini-clusters in TestOzoneFileInterfaces
(#2619)
add e97ea36 HDDS-5679. Use more defensive sizeRequired for replication
manager for container replication. (#2585)
add 2d57e9f HDDS-5708. Skip sending container close command to unhealthy
replica. (#2610)
add 873c0d2 HDDS-5696. Close grpcReplicationClient on exception. (#2595)
add 6543c67 HDDS-5729. support setting maxSizeEnteringTarget and
maxSizeLeavingSource in command line (#2626)
add ec5c51f HDDS-5335. Method not found: allocateBlock - when tracing is
enabled (#2330)
add c5a012b HDDS-5744. Remove unnecessary duplicate semicolon in
PipelineStateMap (#2641)
add ee19b95 HDDS-5748. Reuse mini-clusters in
TestOzoneFSWithObjectStoreCreate (#2646)
add da18617 HDDS-5717. Refactor TestOzoneManagerListVolumes to reuse
mini-ozone clusters (#2615)
add ed30752 HDDS-5749. Reuse mini-clusters in TestOzoneFsHAURLs (#2647)
add d716bf4 HDDS-5402 Support list node based on NodeOperationalState and
NodeState options in printTopology CLI (#2645)
add fab8a12 HDDS-5728. ContainerBalancer should use remaining space to
calculate utilization. (#2625)
add f2c20ce HDDS-5751. Use Mini Cluster Provider to speed up
TestHDDSUpgrade (#2650)
add 5c972c4 HDDS-5753. Split parts of misc suite (#2654)
add 0b9e23e HDDS-5607. remove container manager v1 code (#2525)
add 2d58933 HDDS-5758. Speed up TestKeyInputStream and
TestChunkInputStream by combining some tests (#2656)
add c68468c HDDS-5761. should not shutdown om when setting a bigger
bucket quota than volume quota (#2659)
add 79da6b7 HDDS-5766. Speed up some OM tests by skipping SCM safemode
check (#2662)
add b8afbbc HDDS-5768. Skip safemode check in
TestOzoneManagerRocksDBLogging (#2665)
add 79716f0 HDDS-5765. Test cluster provider possibly returns null (#2663)
add 2c2d2f3 HDDS-5767. Unit check may timeout (#2664)
add b508b25 HDDS-5771. Speed up TestDatanodeHddsVolumeFailureToleration
by reducing dead interval (#2668)
add fc61be5 HDDS-5691. Restrict Recon NSSummaryEndpoint and
ContainerEndpoint to admins. (#2638)
add 3116fa2 HDDS-5773. Avoid code duplication for mini cluster without
datanodes (#2669)
add c25b6d7 HDDS-5759. Bump aspectj version (#2658)
add 87b4971 HDDS-5770. Silent failures of k3s install are difficult to
debug (#2667)
add 605a734 HDDS-5697. SCMContainerPlacementRackAware should both
affinity nodes before fallback (#2597)
add e5c647e HDDS-5370. [FSO] Handle OMClientRequest based on the bucket
layout. (#2533)
add 181266b HDDS-5777. Provide an option to dump table scan data to file.
(#2675)
add 4b3502f HDDS-5792. Speed up TestNSSummaryAdmin by having zero
Datanodes (#2688)
add 2e3c46c HDDS-5782. DatanodeRatisGrpcConfig use wrong type for config
keys. (#2678)
add 041684f HDDS-5789. Enabling container tokens on a upgraded cluster
fails SCM to start up. (#2691)
add af7dc7e HDDS-5800. Bucket usedByte was incorrect when deleting
duplicate keys.
add 8cc6719 HDDS-5523. Fix multipart upload failure in s3 compatibility
tests (#2509)
add aa9eade HDDS-5757. balancer should stop when the cluster can not be
balanced any more (#2676)
add e2c3ac3 HDDS-5747. Reset default values in
ContainerBalancerConfiguration (#2657)
add 5c2ac71 HDDS-5788. Reduce run time for TestOzoneManagerHA tests
(#2689)
add af5b48e HDDS-5636. Use bucket layout specific DB table in
OmMetadataManager:getOpenKeyTable(). (#2561)
add 093f972 HDDS-5787. Provide capability in Ozone to enable ratis
pre-vote (#2680)
add 05eb172 HDDS-5762. remove piplelineStateManager V1 code (#2661)
add c4cbcad HDDS-5710. initialize sequenceIdToLastIdMap when
SequenceIdGenerator#StateManager reinitializes (#2611)
add 53a0009 HDDS-5793. Speed up TestBlockOutputStreamWithFailures by
combining test cases (#2690)
add 1da0bd8 HDDS-5609. Document CI workflow (#2607)
add c886a2f HDDS-5797. Support setting Datanode Reserved Space in
MiniOzoneCluster. (#2695)
add 7bdc1bb HDDS-5799. Speed up TestOzoneFileSystem by using shorter
trash interval (#2699)
add 1ea1d6c HDDS-5808. Update commons-io to 2.11.0 (#2705)
add 186cd6e HDDS-5817. Extract properties for some dependency versions
(#2710)
add 20e8429 HDDS-5815 FileEncryptionInfo is not necessary to be
referenced inside KeyOutputStream. (#2708)
add a59c501 HDDS-5830. Increase idea.max.intellisense.filesize (#2715)
add 8d885b4 HDDS-5557. Cleanup
OzoneBlockTokenSecretManager#validateToken. (#2711)
add ea81ef8 HDDS-5816 Rearrange code and refactor some logic into new
methods in prep for EC addition. (#2709)
add 8eaa9c5 HDDS-4440. Diagrams for old and new approach are mixed up
(#2555)
add c5a92a2 HDDS-5824. `ozone sh volume/bucket/key list` should print
valid JSON array (#2713)
add db97a32 HDDS-5733. Incorrect calculation of iteration related metrics
in ContainerBalancer (#2631)
add efb5263 HDDS-5806. remove BackgroundPipelineCreator v1 code (#2700)
add 57729b4 HDDS-5536. Add metrics for ReplicationSupervisor (#2493)
add 1ffcfe9 HDDS-5349. Avoid usage of locks in listStatus. (#2369)
add 3e05e71 HDDS-5373. [FSO] Define default bucket layout. (#2707)
add f902b13 HDDS-5785. containerCreationLock is a severe bottleneck when
there are many containers to create. (#2679)
add a62c61f HDDS-5823. Change the key permission to 600. (#2722)
add 2f49993 HDDS-5845. Add metrics for stateMachineDataCache cache hit
and miss. (#2726)
add 730a2e8 HDDS-5842. Parse and dump SCM ratis segment file to printable
text (#2731)
add e19fa38 HDDS-5814. Handle InterruptedException in Ozone (#2706)
add ae812c06 HDDS-5831. Remove empty TaskQueue in ContainerStateMachine.
(#2716)
add ed069bd HDDS-5854. Bump json-smart to 2.3.1 (#2736)
add 7376cfe HDDS-5855. Bump commons-compress to 1.21 (#2737)
add 0e5ad31 HDDS-5856. Bump Spring to 5.2.17 (#2738)
add d341c05 HDDS-5857. Bump aws-java-sdk to latest 1.11 (#2740)
add 24ddf0f HDDS-5659. Use bucket layout specific DB table in
OmMetadataManager.getKeyTable() - part1 (#2724)
add bf6da11 HDDS-5858. Fix resource leak due to Files.list (#2741)
add f2f6aa5 HDDS-5671. ContainerBalancer#stop should prevent the current
balancing thread from interrupting itself. (#2578)
add bc81dcf HDDS-5734. add a command to trigger datanode executing du
immediately (#2685)
add a1701e5 HDDS-5850. Update default value of
'ozone.scm.ha.ratis.segment.size' and 'preallocated.size' to improve SCM write
perf. (#2732)
add 7d79d73 HDDS-5812. Increase Ratis version to 2.2.0 (#2753)
add fbb8cf0 HDDS-5794. The misleading "No available thread in pool for
past * second" log message in DN StateContext (#2693)
add 996518a HDDS-5860. Shade Jackson for Ozone Filesystem (#2744)
add 42d3de3 HDDS-5725. Catch possible AlreadyExistsException for create
pipeline command. (#2623)
add 2a4fcde HDDS-5888. Handle InterruptedException in OzoneClientProducer
and ContainerBalancer (#2761)
add ba038e4 HDDS-5875. Do not cache tokens as part of Key Information.
(#2754)
add 2a4bc83 HDDS-5461. Move old objects to delete table on overwrite
(#2433)
add 6e5db67 HDDS-5871. Fix ContainerBalancerConfiguration annotation
string (#2762)
add d8d42b5 HDDS-5868. Change getBucketInfo annotation (#2749)
add 0828afc HDDS-5843. SCM terminates when adding container to a pipeline
during startup (#2745)
add c5aaad8 HDDS-3371. Cleanup of old write-path of key in OM (#2703)
add fc666fa HDDS-5893. Potential data loss on multipart upload commit
(#2763)
add a51822f HDDS-5882 Changes to protobuf messages for HDDS-5881 (#2774)
add f133303 HDDS-5707. List container supports replication factor filter.
(#2606)
add 990db75 HDDS-5835. Follow up task to use bucket layout specific DB
table in OmMetadataManager.getKeyTable() - part2 (#2751)
add 0906337 HDDS-5902 Ambiguous OM kerberos keytab settings (#2769)
add a5d6707 HDDS-5901. Delete on volume/bucket throws fatal error on ofs
(#2776)
add 26824ca HDDS-5884. OM Validate S3 Auth for write requests. (#2778)
add d018d00 HDDS-5805. remove containerStateManager V1 code (#2727)
add f153236 HDDS-5863. Error message having null fields on volume
creation (#2752)
add 5c76856 HDDS-5885. OM Auth Validate for read requests and handle non
ratis enabled code. (#2779)
add 5f0a588 HDDS-5908. MPU getKey can fail, if completeMPU result is
still in cache. (#2787)
add e427b71 HDDS-5883 Change S3G client to set S3 Auth per req (#2775)
add 73a081a HDDS-5864. Retry when DN connection issue during
getBlock/ReadChunk call during Ozone key Read (#2746)
add 83f8b04 HDDS-5736. Update navbar from Hadoop to Ozone on doc page
(#2792)
add 47cb26a HDDS-3983 Ozone RocksDB Iterator wrapper should not expose
key() and value() API. (#2402)
add 565972c Support more detailed error log when handleFlush in Ozone
client (#2795)
add 1cf43ac HDDS-5931. Broken link in dist/README (#2799)
add 9cea16d HDDS-5933. Ozone Filesystem shaded jars include unnecessary
dependencies (#2798)
add eb7136d HDDS-3369. Cleanup old write-path of volume in OM. (#2780)
add c088cd1 HDDS-5925. Fix SecuringOzoneHTTP doc (#2796)
add ac86851 HDDS-5943. Shutdown ResultHandlerExecutorService for
StorageVolumeChecker. (#2806)
add aa7e083 HDDS-5944. Fix typo in OzoneManagerServiceProviderImpl.java
(#2807)
add 6ebb316 HDDS-5945. bump rocksdb version to 6.25.3 (#2809)
add 4c2181f HDDS-5935. Bump Spring to 5.2.18 (#2801)
add f3b482a HDDS-5922. ignore deletetransaction when container is not
found (#2793)
add c63479c HDDS-5839. Make sure buckets created from OFS are in
FILE_SYSTEM_OPTIMIZED layout (#2730)
add 96e2d7f HDDS-5947. Remove unused mina-core and sshd-core dependencies
(#2811)
add 252fee4 HDDS-5910 Add additional verification for S3 Auth. (#2817)
add 9d9e780 HDDS-5956. Speed up
TestOzoneRpcClientAbstract#testZReadKeyWithUnhealthyContainerReplica (#2820)
add 0eedab9 HDDS-5929. Make FSO and OBS bucket layouts independent of
normalization config flag (#2819)
add ba3ac30 HDDS-5960. Change option name to bucketlayout instead of type
(#2825)
add 4558e40 HDDS-5970. Remove OMKeyRequest#getBucketLayout overridden
method in subclasses (#2830)
add 9384bd7 HDDS-5820. Intermittent failure in
TestPipelinePlacementPolicy#testPickLowestLoadAnchor (#2757)
add d0e9140 HDDS-5958. Refine container scrub log message. (#2823)
add 45c1899 HDDS-5937. Inaccurate bucket info returned from bucket list
command in Shell (#2832)
add c2db377 HDDS-5872. Do not failover on some RpcExceptions (#2772)
add 71f47d4 HDDS-5981. Refactor usage of bucket type. (#2836)
add 026d878 HDDS-5982. Remove isBucketFSOptimized flag. (#2838)
add 346b55d HDDS-5906. Fix DBScanner to support Datanode DB (#2781)
add 1b7072c HDDS-5983. Prefix Parser tool should only work for FSO
buckets. (#2839)
add 5ef59b4 HDDS-5957. Bump proto-backwards-compatibility from 1.0.5 to
1.0.7 (#2821)
add b4a785c HDDS-5891. OFS mkdir -p does not work as expected for bucket
creation when volume exists due to volume create ACL check (#2814)
add 6ea9916 HDDS-5964. [S3-TDE] Range Reads not working on encrypted
buckets (#2827)
add 83e5aa3 HDDS-5848. Introduce more replication metrics (#2758)
add 4348d1d HDDS-5980. Get on a key which is created/overwrite through
MPU on a TDE bucket, the data does not match with uploaded data. (#2844)
add 78e725c HDDS-5969. TestRootedOzoneFileSystem parameterized test is
not initialized properly (#2843)
add 281f3b0 HDDS-5870. Make Datanode usageinfo command output more
readable (#2816)
add 07b54e2 HDDS-5534. Verify config is updated on all OMs before
proceeding with Bootstrap (#2491)
add 722fd59 HDDS-5898 S3G in secure mode checks OM version. (#2822)
add 52e619c HDDS-5517. Support multiple container moves from a source
datanode in one balance iteration (#2808)
add 6372776 HDDS-5976. add raftlog IOException notifier for SCM HA (#2840)
add bc6029e [FSO] Allow existing/old buckets with any layout during OM
startup (#2571)
add ffe53e7 HDDS-6019. Fix s3g create key error in cluster with
replication one. (#2855)
add f780851 HDDS-6003. Update Ozone master branch version after 1.2.0
release (#2863)
add 4f56b1a HDDS-6043. Buckets created via link command do not mirror
layout of s… (#2867)
add 7afd64f HDDS-5897. Support configuration for including/excluding
datanodes for balancing (#2786)
add 97d900e HDDS-5989. Binary content printed for failed chunk write
(#2846)
add 1e76ffc HDDS-6011. Freon datanode chunk generator fails with NPE in
secure cluster (#2853)
add 5472ea4 HDDS-5497. Save log from init containers (#2547)
add 8b4d4a9 HDDS-5804. SCMContainerPlacementCapacity to choose lower
utilized node (#2784)
add 4e91226 HDDS-5917. S3 acceptance test failure due to too wide
assertion (#2789)
add 5b0604d HDDS-5702. SCM remote benchmark tool. (#2600)
add 2341c7c HDDS-5936. Duplicate Test Ozone Shell invocation (#2803)
add 1892230 HDDS-6054. Fix resource leak due to OzoneManager.createOm
(#2876)
add ec77f6c HDDS-6000. Freon datanode chunk validator fails checksum
validation (#2850)
add 090672d HDDS-6055. Refactor isFileSystemOptimized usage in
OzoneManagerUtils (#2877)
add b69a3d1 HDDS-5959. Handles bucket layout validation logic in ofs/o3fs
client. (#2837)
add b8e1e8f HDDS-5999. Freon datanode chunk validator does not find
pipeline from param (#2879)
add 3012cd1 HDDS-6014. Add proto.lock files from ozone-1.2 release branch
to master (#2865)
add ef1f3d3 HDDS-5962. Limit grpc threads in XceiverServerGrpc. (#2824)
add d6e0e47 HDDS-5997 Centralize string based replication config
validation via ReplicationConfigValidator (#2871)
add 8ef63ec HDDS-6058. Bump aws-java-sdk to latest 1.12 (#2881)
add 66c581d HDDS-6041. [FSO] Revisit robot test cases related to
prefix/simple me… (#2885)
add ab3b731 HDDS-6061. Peer datanode cannot add group for pipeline in
secure env (#2882)
add e86119f HDDS-5965. Recon should be able to distinguish between
containers that have no replicas and those have all replicas as UNHEALTHY.
(#2874)
add 9c4bb72 HDDS-5973. Changed sequence of steps during pipeline close.
(#2847)
add df0489a HDDS-5903. Add Support for Bucket Owner Acls (#2826)
add 3097d2b HDDS-6059. Change Cannot close container log message from
WARN to DEBUG (#2890)
add ea53dc1 HDDS-3227. Ensure eviction of stateMachineData from cache
only when both followers catch up (#2704)
add 46c95ba HDDS-6067. Improve Debugging around unhealthy container state
on Datanode. (#2888)
add 937a3d0 HDDS-6071. ResourceLimitCache leaks permits (#2895)
add 176ef94 HDDS-5385. [FSO] Remove ozone.om.metadata.layout config in OM
(#2887)
add ab8cc32 HDDS-6073. Document: Fix images alignment in the doc pages
(#2898)
add d972c7b HDDS-6069: Fix XmlRootElement for CopyObjectResponse (#2891)
add 41f6844 HDDS-6046. Multi-threaded validator for freon
RandomKeyGenerator. (#2869)
add 8532e08 HDDS-6078. Move BucketManagerImpl#resolveLinkBucketLayout
into utilit… (#2902)
add 831b195 HDDS-5974. Remove OMClientResponse#getBucketLayout override
in subclasses - part1 (#2894)
add 173f46a HDDS-6085. Update log4j to 2.15 (#2906)
add 5dd1144 HDDS-6066. Cleanup OmService injection. (#2896)
add 0bc100a HDDS-6090. Collect output.log from acceptance tests (#2908)
add 8587de7 HDDS-6096. S3 copy object fails for source key with special
char (#2912)
add dfcc69c HDDS-6018 Closing state to quasiclosed (#2854)
add 09b06eb HDDS-6104. Update log4j version to 2.16.0 version. (#2920)
add 4d1b5da HDDS-6053. Fix too short container scrubber data scan
interval. (#2875)
add e1bef2c HDDS-6080. Simplify the logic in
SimpleContainerDownloader.getContainerDataFromReplicas (#2903)
add 47cd204 HDDS-6107. Exclude hugo lock file from source tarball (#2923)
add c4579ec HDDS-6089. Remove OMClientResponse#getBucketLayout override
in subclasses - part2 (#2915)
add b75ec9d HDDS-6072. Fix increased integration test execution time
(#2900)
add 56d0ebb HDDS-6105. remove db cache size config (#2922)
add af4730a HDDS-5912. Add description of hdds.datanode.dir in Getting
Started (#2783)
add 3fa3f36 HDDS-6110. Add hugo lock file to gitignore (#2928)
add 4770eab HDDS-5795. [FSO] Add unit test case with mpu open key during
abort. (#2931)
add fe7f5b4 HDDS-5490. Remove (Decommission) an OM node from HA ring
(#2886)
add 8ceb6cb HDDS-6124. Update log4j version to 2.17.0 version (#2933)
add 69e8617 HDDS-6125. Support Hugo 0.91.0. (#2934)
add e01b471 HDDS-5602. make it configurable to choose the nearest one as
the target in the candidates according to networkTopology (#2756)
add 26a9cdb HDDS-6101. use FileUtils.moveDirectory instead of Files.move
when installing snapshot (#2918)
add 6cd9211 HDDS-5975. Serve url-encoded key and prefix name for
ListObjectResponse (#2834)
add 9dc63a3 HDDS-5807. [FSO] Merge HDDS-4653 changes to support TDE for
MPU Keys in FSO bucket layout. (#2941)
add 97143b8 HDDS-6129. OM has thread unsafe issues that can cause OM
shutdown. (#2938)
add b570d0a HDDS-6070. ContainerBalancerConfig doesn't read config from
ozone-site.xml (#2893)
add 4ed3ef8 HDDS-6121. DatanodeAdminMonitor should log detailed
information for a limited number of containers (#2932)
add 9103aa0 HDDS-5993. Disable flaky
TestRandomKeyGenerator#cleanObjectsTest
add a5cc886 HDDS-5319 Intermittent failure in
TestSCMUpdateServiceGrpcServer (#2558)
add d6cbb15 HDDS-6143. Update log4j version to 2.17.1 (#2952)
add b5c5853 HDDS-6142. Remove unused dependency: jdom (#2951)
add cb59f01 HDDS-6119. Reset the property name to configure the datanode
kerberos keytab (#2949)
add e6179c1 HDDS-6118. Documentation missing settings to secure S3g with
Kerberos (#2950)
add 144e982 HDDS-6108. Add slf4j and log4j.properties for tests in
hadoop-ozone/client (#2925)
add b7496c9 HDDS-6146. TestDefaultCAServer#testIntermediaryCA failure
(#2955)
add 4d3f8ba HDDS-6103. Avoid refresh pipeline for key override in
Key/File Create. (#2926)
add 644ef95 HDDS-6145. Include docker-compose output in acceptance test
output.log (#2958)
add 30cb5e7 HDDS-6123. Disable system exit in integration tests (#2959)
add 4d0bf19 Commit for merge requested in PR #2945 to merge Apache Ozone
master branch with HDDS-5881 changes INTO HDDS-4400-s3-performance feature
branch. Merge allows s3 gateway gRPC to work with s3 gateway persistent
connection changes made in HDDS-5881 found in master branch.
add abb3637 Synchronization added to getOzoneConfiguration for
OzoneClientProducer to satisfy CI findbugs.
No new revisions were added by this update.
Summary of changes:
.github/ci.md | 102 ++
.github/workflows/post-commit.yml | 6 +-
.gitignore | 1 +
CONTRIBUTING.md | 4 +-
hadoop-hdds/client/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 38 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 52 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 +
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 2 +-
.../storage/DummyBlockInputStreamWithRetry.java | 14 +-
.../hdds/scm/storage/TestBlockInputStream.java | 32 +-
hadoop-hdds/common/pom.xml | 4 +-
.../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 4 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 122 ++
.../java/org/apache/hadoop/hdds/NodeDetails.java | 0
.../hadoop/hdds/client/RatisReplicationConfig.java | 27 +-
.../hdds/client/ReplicatedReplicationConfig.java | 18 +-
.../hadoop/hdds/client/ReplicationConfig.java | 138 +-
.../hdds/client/ReplicationConfigValidator.java | 5 +-
.../hdds/client/StandaloneReplicationConfig.java | 23 +-
.../hadoop/hdds/conf/OzoneConfiguration.java | 5 +-
.../hadoop/hdds/fs/CachingSpaceUsageSource.java | 23 +-
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 1 +
.../apache/hadoop/hdds/recon/ReconConfigKeys.java | 20 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 22 +-
.../apache/hadoop/hdds/scm/XceiverClientSpi.java | 6 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 11 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 7 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 33 +-
.../hadoop/hdds/scm/net/NetworkTopology.java | 3 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 29 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 2 +-
.../protocol/StorageContainerLocationProtocol.java | 24 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 5 -
.../apache/hadoop/hdds/tracing/TraceAllMethod.java | 23 +-
.../apache/hadoop/hdds/tracing/TracingUtil.java | 6 +-
.../hadoop/hdds/utils/ResourceLimitCache.java | 7 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 12 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 -
.../upgrade/AbstractLayoutVersionManager.java | 158 ++-
.../common/src/main/resources/ozone-default.xml | 183 +--
.../hadoop/hdds/client/TestReplicationConfig.java | 280 ++--
.../hdds/scm/net/TestNetworkTopologyImpl.java | 33 +-
.../hadoop/hdds/tracing/TestTraceAllMethod.java | 68 +
.../hadoop/hdds/tracing/TestTracingUtil.java | 49 +
.../hadoop/hdds/utils/TestResourceLimitCache.java | 46 +-
hadoop-hdds/config/pom.xml | 4 +-
hadoop-hdds/container-service/pom.xml | 4 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 11 +-
.../container/common/helpers/ContainerUtils.java | 106 +-
.../ozone/container/common/impl/ContainerSet.java | 3 +
.../container/common/impl/HddsDispatcher.java | 5 +-
.../common/interfaces/ContainerDispatcher.java | 7 +-
.../ozone/container/common/interfaces/Handler.java | 2 +-
.../common/statemachine/DatanodeConfiguration.java | 64 +
.../common/statemachine/DatanodeStateMachine.java | 21 +-
.../common/statemachine/StateContext.java | 14 +-
.../CloseContainerCommandHandler.java | 15 +-
.../ClosePipelineCommandHandler.java | 13 +-
.../CreatePipelineCommandHandler.java | 10 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 4 +-
.../RefreshVolumeUsageCommandHandler.java | 73 +
.../states/endpoint/VersionEndpointTask.java | 5 +-
.../common/transport/server/XceiverServerGrpc.java | 52 +-
.../common/transport/server/ratis/CSMMetrics.java | 10 +
.../server/ratis/ContainerStateMachine.java | 93 +-
.../transport/server/ratis/XceiverServerRatis.java | 17 +-
.../container/common/utils/HddsVolumeUtil.java | 72 +-
.../ozone/container/common/volume/HddsVolume.java | 2 +-
.../container/common/volume/MutableVolumeSet.java | 17 +-
.../container/common/volume/StorageVolume.java | 4 +
.../common/volume/StorageVolumeChecker.java | 1 +
.../ozone/container/common/volume/VolumeInfo.java | 13 +-
.../ozone/container/common/volume/VolumeUsage.java | 3 +
.../container/keyvalue/KeyValueContainer.java | 16 +-
.../container/keyvalue/KeyValueContainerCheck.java | 6 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 62 +-
.../container/keyvalue/helpers/ChunkUtils.java | 18 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 9 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 9 +-
.../metadata/DatanodeSchemaOneDBDefinition.java | 6 +-
.../container/ozoneimpl/ContainerController.java | 24 +-
.../ozone/container/ozoneimpl/ContainerReader.java | 161 +--
.../ozoneimpl/ContainerScrubberConfiguration.java | 65 +-
.../ozone/container/ozoneimpl/OzoneContainer.java | 16 +-
.../container/replication/ContainerDownloader.java | 3 +-
.../replication/DownloadAndImportReplicator.java | 16 +-
.../replication/GrpcReplicationClient.java | 4 +-
.../container/replication/MeasuredReplicator.java | 34 +-
.../container/replication/ReplicationServer.java | 7 +-
.../replication/ReplicationSupervisor.java | 9 +-
.../replication/ReplicationSupervisorMetrics.java | 71 +
.../replication/SimpleContainerDownloader.java | 56 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 5 +-
.../upgrade/DatanodeMetadataFeatures.java | 53 -
.../ScmHAFinalizeUpgradeActionDatanode.java | 129 ++
.../ScmHAFirstUpgradeLayoutChangeAction.java | 45 -
.../upgrade/VersionedDatanodeFeatures.java | 145 ++
.../hadoop/ozone/container/common/ScmTestMock.java | 9 +
.../ozone/container/common/TestContainerCache.java | 23 +-
.../common/TestKeyValueContainerData.java | 8 +-
.../common/helpers/TestContainerUtils.java | 2 +-
.../common/impl/TestContainerDataYaml.java | 41 +-
.../TestCloseContainerCommandHandler.java | 37 +-
.../common/volume/TestStorageVolumeChecker.java | 4 +
.../TestContainerScrubberConfiguration.java | 97 ++
.../replication/TestMeasuredReplicator.java | 34 +-
.../replication/TestReplicationSupervisor.java | 63 +-
.../replication/TestSimpleContainerDownloader.java | 13 +-
.../upgrade/TestDatanodeUpgradeToScmHA.java | 741 ++++++++++
hadoop-hdds/docs/config.yaml | 5 +
hadoop-hdds/docs/content/concept/Containers.md | 2 +-
hadoop-hdds/docs/content/concept/Datanodes.md | 4 +-
hadoop-hdds/docs/content/concept/OzoneManager.md | 8 +-
.../docs/content/concept/OzoneManager.zh.md | 2 +-
hadoop-hdds/docs/content/concept/Recon.md | 6 +-
hadoop-hdds/docs/content/feature/OM-HA.md | 5 +-
hadoop-hdds/docs/content/feature/PrefixFSO.md | 24 +-
hadoop-hdds/docs/content/feature/SCM-HA.md | 6 +-
hadoop-hdds/docs/content/feature/Topology.md | 2 +-
hadoop-hdds/docs/content/interface/CSI.md | 4 +-
hadoop-hdds/docs/content/interface/CSI.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Cli.md | 7 +-
hadoop-hdds/docs/content/interface/ReconApi.md | 238 +++-
hadoop-hdds/docs/content/interface/ReconApi.zh.md | 2 +-
hadoop-hdds/docs/content/security/SecureOzone.md | 10 +-
.../docs/content/security/SecureOzone.zh.md | 34 +-
.../docs/content/security/SecuringDatanodes.md | 2 +-
.../docs/content/security/SecuringDatanodes.zh.md | 2 +-
.../docs/content/security/SecuringOzoneHTTP.md | 20 +-
hadoop-hdds/docs/content/start/FromSource.md | 2 +-
hadoop-hdds/docs/content/start/OnPrem.md | 1 +
hadoop-hdds/docs/content/start/OnPrem.zh.md | 2 +
hadoop-hdds/docs/pom.xml | 4 +-
.../docs/static/design/s3-performance-new.png | Bin 33688 -> 34927 bytes
.../docs/static/design/s3-performance-old.png | Bin 34927 -> 33688 bytes
.../docs/themes/ozonedoc/layouts/index.html | 2 +
.../themes/ozonedoc/layouts/partials/navbar.html | 6 +-
.../themes/ozonedoc/layouts/partials/sidebar.html | 4 +-
.../layouts/{index.html => shortcodes/image.html} | 24 +-
hadoop-hdds/framework/pom.xml | 4 +-
.../java/org/apache/hadoop/hdds/ExitManager.java | 16 +-
.../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java | 4 +-
.../hdds/conf/DatanodeRatisServerConfig.java | 16 +
.../scm/protocol/ScmBlockLocationProtocol.java | 2 +-
...inerLocationProtocolClientSideTranslatorPB.java | 31 +-
.../certificate/client/DNCertificateClient.java | 5 -
.../client/DefaultCertificateClient.java | 2 +-
.../certificate/client/OMCertificateClient.java | 5 -
.../certificate/client/SCMCertificateClient.java | 10 +-
.../hadoop/hdds/security/x509/keys/KeyCodec.java | 31 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 16 +-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 2 +-
.../hadoop/hdds/utils/db/RDBStoreIterator.java | 19 +-
.../apache/hadoop/hdds/utils/db/TableIterator.java | 12 -
.../apache/hadoop/hdds/utils/db/TypedTable.java | 18 -
.../certificate/authority/TestDefaultCAServer.java | 6 +-
.../hdds/security/x509/keys/TestKeyCodec.java | 5 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 19 +-
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
.../src/main/proto/ScmAdminProtocol.proto | 3 +
.../interface-admin/src/main/resources/proto.lock | 504 +++++++
hadoop-hdds/interface-client/pom.xml | 4 +-
.../src/main/proto/DatanodeClientProtocol.proto | 1 +
.../interface-client/src/main/proto/hdds.proto | 11 +
.../interface-client/src/main/resources/proto.lock | 408 +++++-
hadoop-hdds/interface-server/pom.xml | 4 +-
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 +
.../interface-server/src/main/resources/proto.lock | 1024 ++++++++++++++
hadoop-hdds/pom.xml | 4 +-
hadoop-hdds/server-scm/pom.xml | 4 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 13 +-
.../scm/block/DeletedBlockLogStateManagerImpl.java | 22 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 9 +-
.../container/AbstractContainerReportHandler.java | 6 +-
.../scm/container/CloseContainerEventHandler.java | 16 +-
.../hdds/scm/container/ContainerManager.java | 144 +-
.../hdds/scm/container/ContainerManagerImpl.java | 75 +-
.../hdds/scm/container/ContainerManagerV2.java | 190 ---
.../hdds/scm/container/ContainerReportHandler.java | 6 +-
.../hdds/scm/container/ContainerStateManager.java | 570 ++------
.../scm/container/ContainerStateManagerImpl.java | 19 +-
.../scm/container/ContainerStateManagerV2.java | 189 ---
.../IncrementalContainerReportHandler.java | 2 +-
.../hdds/scm/container/ReplicationManager.java | 74 +-
.../hdds/scm/container/SCMContainerManager.java | 672 ----------
.../balancer/AbstractFindTargetGreedy.java | 245 ++++
.../scm/container/balancer/ContainerBalancer.java | 413 +++---
.../balancer/ContainerBalancerConfiguration.java | 156 ++-
.../balancer/ContainerBalancerMetrics.java | 115 +-
.../ContainerBalancerSelectionCriteria.java | 34 +-
.../scm/container/balancer/FindSourceGreedy.java | 158 +++
.../scm/container/balancer/FindSourceStrategy.java | 67 +
.../scm/container/balancer/FindTargetGreedy.java | 135 --
.../FindTargetGreedyByNetworkTopology.java | 79 ++
.../balancer/FindTargetGreedyByUsageInfo.java | 52 +
.../scm/container/balancer/FindTargetStrategy.java | 32 +-
.../algorithms/SCMContainerPlacementCapacity.java | 2 +-
.../algorithms/SCMContainerPlacementRackAware.java | 72 +-
.../container/placement/metrics/SCMNodeStat.java | 23 -
.../replication/ReplicationManagerMetrics.java | 85 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 18 +-
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 23 +-
.../apache/hadoop/hdds/scm/ha/SCMRatisRequest.java | 24 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 7 +
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 26 +-
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 42 +
.../hadoop/hdds/scm/node/DatanodeUsageInfo.java | 65 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 6 +-
.../hdds/scm/node/NodeDecommissionManager.java | 8 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 8 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 60 +-
.../scm/pipeline/BackgroundPipelineCreator.java | 266 +++-
.../scm/pipeline/BackgroundPipelineCreatorV2.java | 332 -----
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 2 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 30 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 105 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 6 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 271 ++--
...erV2Impl.java => PipelineStateManagerImpl.java} | 70 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 29 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +-
.../hdds/scm/pipeline/RatisPipelineUtils.java | 20 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +-
.../hadoop/hdds/scm/pipeline/StateManager.java | 127 --
.../pipeline/WritableRatisContainerProvider.java | 105 +-
.../algorithms/DefaultLeaderChoosePolicy.java | 4 +-
.../choose/algorithms/LeaderChoosePolicy.java | 8 +-
.../algorithms/LeaderChoosePolicyFactory.java | 8 +-
.../algorithms/MinLeaderCountChoosePolicy.java | 6 +-
...lockLocationProtocolServerSideTranslatorPB.java | 2 +-
...inerLocationProtocolServerSideTranslatorPB.java | 21 +-
.../hdds/scm/safemode/ContainerSafeModeRule.java | 10 +-
.../hdds/scm/safemode/SCMSafeModeManager.java | 10 +-
.../scm/server/OzoneStorageContainerManager.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 76 +-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 8 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 3 +-
.../hdds/scm/server/StorageContainerManager.java | 89 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 5 +-
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 30 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 6 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 22 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 5 +
.../container/TestCloseContainerEventHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 8 +-
.../scm/container/TestContainerReportHandler.java | 269 ++--
.../scm/container/TestContainerStateManager.java | 91 +-
.../TestIncrementalContainerReportHandler.java | 130 +-
.../hdds/scm/container/TestReplicationManager.java | 405 +++---
.../scm/container/TestSCMContainerManager.java | 324 -----
.../scm/container/TestUnknownContainerReport.java | 51 +-
.../container/balancer/TestContainerBalancer.java | 191 ++-
.../container/balancer/TestFindTargetStrategy.java | 196 +++
.../hdds/scm/container/balancer}/package-info.java | 13 +-
.../TestSCMContainerPlacementCapacity.java | 6 +-
.../TestSCMContainerPlacementRackAware.java | 42 +
.../hdds/scm/ha/TestReplicationAnnotation.java | 8 +-
.../hdds/scm/node/TestContainerPlacement.java | 122 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 14 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 11 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 53 +-
.../scm/pipeline/MockRatisPipelineProvider.java | 8 +-
.../TestPipelineDatanodesIntersection.java | 52 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 85 +-
.../scm/pipeline/TestPipelinePlacementPolicy.java | 73 +-
...ager.java => TestPipelineStateManagerImpl.java} | 245 +++-
.../scm/pipeline/TestRatisPipelineProvider.java | 66 +-
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 209 ---
.../scm/pipeline/TestSimplePipelineProvider.java | 48 +-
.../choose/algorithms/TestLeaderChoosePolicy.java | 6 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 8 +-
.../ozone/container/common/TestEndPoint.java | 4 +-
.../placement/TestContainerPlacement.java | 4 +-
.../testutils/ReplicationNodeManagerMock.java | 11 +
hadoop-hdds/test-utils/pom.xml | 4 +-
hadoop-hdds/tools/pom.xml | 4 +-
.../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 18 +
.../scm/cli/ContainerBalancerStartSubcommand.java | 13 +-
.../hdds/scm/cli/ContainerOperationClient.java | 12 +-
.../hadoop/hdds/scm/cli/TopologySubcommand.java | 46 +-
.../hdds/scm/cli/container/ListSubcommand.java | 6 +-
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 41 +-
.../datanode/TestContainerBalancerSubCommand.java | 6 +-
hadoop-ozone/client/pom.xml | 14 +-
.../org/apache/hadoop/ozone/client/BucketArgs.java | 29 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 40 +-
.../ozone/client/io/BlockOutputStreamEntry.java | 268 ++--
.../client/io/BlockOutputStreamEntryPool.java | 109 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 16 +-
.../hadoop/ozone/client/io/OzoneInputStream.java | 33 +-
.../ozone/client/protocol/ClientProtocol.java | 20 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 121 +-
.../hadoop/ozone/client/rpc/RpcClientTest.java | 217 +++
.../src/test/resources/log4j.properties | 0
hadoop-ozone/common/pom.xml | 8 +-
.../main/java/org/apache/hadoop/ozone/OFSPath.java | 48 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 145 +-
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 40 +-
.../ozone/om/ha/OMFailoverProxyProvider.java | 14 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 25 +
.../hadoop/ozone/om/helpers/OMNodeDetails.java | 67 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 32 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 63 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 17 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 24 +-
.../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 4 +
.../hadoop/ozone/om/helpers/ServiceInfo.java | 41 +-
.../hadoop/ozone/om/protocol/OMAdminProtocol.java | 29 +-
.../hadoop/ozone/om/protocol/OMConfiguration.java | 104 ++
.../ozone/om/protocol/OzoneManagerProtocol.java | 168 ++-
.../apache/hadoop/ozone/om/protocol/S3Auth.java} | 39 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 37 -
.../protocolPB/OMAdminProtocolClientSideImpl.java | 224 ++++
.../ozone/om/protocolPB/OMAdminProtocolPB.java} | 42 +-
.../om/protocolPB/OzoneManagerClientProtocol.java | 27 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 63 +-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 41 +-
hadoop-ozone/csi/pom.xml | 4 +-
.../org/apache/hadoop/ozone/csi/NodeService.java | 10 +-
hadoop-ozone/datanode/pom.xml | 4 +-
hadoop-ozone/dev-support/checks/_lib.sh | 8 +-
hadoop-ozone/dev-support/checks/acceptance.sh | 5 +-
hadoop-ozone/dev-support/checks/kubernetes.sh | 4 +-
hadoop-ozone/dist/README.md | 6 +-
hadoop-ozone/dist/pom.xml | 4 +-
.../dist/src/main/assemblies/ozone-src.xml | 1 +
.../dist/src/main/compose/ozone-ha/test.sh | 2 +
.../src/main/compose/ozone-mr/hadoop27/test.sh | 2 +
.../src/main/compose/ozone-mr/hadoop31/test.sh | 2 +
.../src/main/compose/ozone-mr/hadoop32/test.sh | 2 +
.../src/main/compose/ozone-mr/hadoop33/test.sh | 2 +
.../dist/src/main/compose/ozone-mr/test.sh | 3 +
hadoop-ozone/dist/src/main/compose/ozone/README.md | 6 -
.../src/main/compose/ozone/docker-compose.yaml | 10 -
.../dist/src/main/compose/ozone/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/ozone/run.sh | 4 -
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 29 +-
.../compose/ozonesecure-ha/docker-compose.yaml | 2 +
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure-ha/test.sh | 2 +
.../src/main/compose/ozonesecure-mr/docker-config | 5 +-
.../dist/src/main/compose/ozonesecure-mr/test.sh | 2 +
.../src/main/compose/ozonesecure/docker-config | 5 +-
hadoop-ozone/dist/src/main/compose/test-all.sh | 3 +-
hadoop-ozone/dist/src/main/compose/testlib.sh | 23 +-
.../dist/src/main/compose/upgrade/README.md | 35 +-
.../compose/upgrade/delete-and-regenerate-data.sh | 53 -
.../dist/src/main/compose/upgrade/delete-data.sh | 36 -
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 9 +-
.../manual-upgrade/0.5.0-1.1.0/callback.sh | 40 -
.../upgrades/manual-upgrade}/README.md | 7 +-
.../{1.0.0-1.1.0 => 1.1.0-1.2.0}/callback.sh | 17 +-
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 31 +-
.../dist/src/main/compose/versions/0.5.0.sh | 26 -
.../dist/src/main/compose/versions/1.0.0.sh | 26 -
.../dist/src/main/compose/versions/1.1.0.sh | 26 -
.../dist/src/main/compose/xcompat/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 4 -
hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +-
.../src/main/smoketest/admincli/container.robot | 8 +
.../dist/src/main/smoketest/basic/links.robot | 8 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 10 +-
.../main/smoketest/basic/ozone-shell-single.robot | 2 +-
.../src/main/smoketest/basic/ozone-shell.robot | 15 +-
.../dist/src/main/smoketest/createmrenv.robot | 2 +-
.../dist/src/main/smoketest/freon/generate.robot | 9 +
.../dist/src/main/smoketest/freon/validate.robot | 9 +
.../dist/src/main/smoketest/omha/testOMHA.robot | 2 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 18 +-
.../dist/src/main/smoketest/ozonefs/setup.robot | 14 +-
.../dist/src/main/smoketest/recon/recon-api.robot | 53 +-
.../src/main/smoketest/s3/MultipartUpload.robot | 23 +-
.../dist/src/main/smoketest/s3/awss3.robot | 8 +
.../dist/src/main/smoketest/s3/objectcopy.robot | 20 +-
.../dist/src/main/smoketest/s3/objectdelete.robot | 34 +-
.../src/main/smoketest/s3/objectmultidelete.robot | 24 +-
.../dist/src/main/smoketest/s3/objectputget.robot | 42 +-
.../main/smoketest/security/ozone-secure-fs.robot | 12 +-
.../dist/src/main/smoketest/topology/cli.robot | 6 +
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../fault-injection-test/network-tests/pom.xml | 2 +-
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 4 +-
.../dev-support/findbugsExcludeFile.xml | 4 +
hadoop-ozone/integration-test/pom.xml | 4 +-
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 19 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 4 +
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 33 +-
.../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 152 ++-
.../fs/ozone/TestOzoneFileInterfacesWithFSO.java | 29 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 47 +-
.../fs/ozone/TestOzoneFileSystemMetrics.java | 3 +
.../fs/ozone/TestOzoneFileSystemMissingParent.java | 2 +
.../fs/ozone/TestOzoneFileSystemPrefixParser.java | 9 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 9 +-
.../fs/ozone/TestOzoneFileSystemWithLinks.java | 248 ++++
.../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 40 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 161 ++-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 12 +-
.../hadoop/fs/ozone/contract/OzoneContract.java | 5 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 4 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 4 +-
.../TestContainerStateManagerIntegration.java | 138 +-
.../metrics/TestSCMContainerManagerMetrics.java | 4 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 4 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 4 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 4 +-
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 6 +-
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 43 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 40 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 9 +-
.../hadoop/ozone/MiniOzoneClusterProvider.java | 286 ++++
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 150 ++-
.../ozone/TestContainerBalancerOperations.java | 8 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 16 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 15 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 +
.../hadoop/ozone/TestStandardOutputUtil.java | 84 ++
.../rpc/TestBlockOutputStreamWithFailures.java | 20 +-
...estBlockOutputStreamWithFailuresFlushDelay.java | 20 +-
.../rpc/TestContainerStateMachineFailures.java | 103 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 38 +-
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 31 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 140 +-
.../TestOzoneRpcClientWithKeyLatestVersion.java | 209 +++
.../hadoop/ozone/client/rpc/TestReadRetries.java | 14 +-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 100 +-
.../client/rpc/read/TestChunkInputStream.java | 13 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 44 -
.../ozone/client/rpc/read/TestKeyInputStream.java | 58 +-
.../ozone/container/TestContainerReplication.java | 47 -
.../apache/hadoop/ozone/container/TestHelper.java | 4 +-
.../commandhandler/TestBlockDeletion.java | 10 +-
.../transport/server/ratis/TestCSMMetrics.java | 7 +
.../container/ozoneimpl/TestOzoneContainer.java | 8 +-
.../ozoneimpl/TestSecureOzoneContainer.java | 2 +
.../server/TestSecureContainerServer.java | 3 +
.../ozone/dn/TestDatanodeLayoutUpgradeTool.java | 131 --
.../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 4 +-
.../TestDatanodeHddsVolumeFailureToleration.java | 40 +-
.../ozone/freon/TestHadoopDirTreeGenerator.java | 5 +-
.../freon/TestHadoopDirTreeGeneratorWithFSO.java | 4 +-
.../ozone/freon/TestHadoopNestedDirGenerator.java | 3 +
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 4 +-
.../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 423 ++++++
.../apache/hadoop/ozone/om/TestBucketOwner.java | 244 ++++
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 44 +-
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 10 +-
.../ozone/om/TestOMStartupWithBucketLayout.java | 161 +++
.../apache/hadoop/ozone/om/TestObjectStore.java | 233 ++++
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 26 +-
.../org/apache/hadoop/ozone/om/TestOmAcls.java | 6 +-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 27 +-
.../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 64 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 301 +++--
.../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 216 ---
.../ozone/om/TestOzoneManagerConfiguration.java | 2 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 114 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +-
.../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 24 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 130 +-
.../ozone/om/TestOzoneManagerHAWithFailover.java | 65 +
.../ozone/om/TestOzoneManagerListVolumes.java | 117 +-
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 28 +-
.../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +-
.../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 3 +-
.../apache/hadoop/ozone/om/TestScmSafeMode.java | 7 +-
.../om/ratis/TestOzoneManagerRatisRequest.java | 71 +
.../TestOzoneHARatisLogParser.java} | 52 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 10 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 4 +-
.../ozone/recon/TestReconWithOzoneManager.java | 19 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 6 +-
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 4 +-
.../hadoop/ozone/scm/TestXceiverClientManager.java | 18 +-
.../scm/node/TestDecommissionAndMaintenance.java | 128 +-
.../hadoop/ozone/shell/TestNSSummaryAdmin.java | 175 +++
.../hadoop/ozone/shell/TestOzoneShellHA.java | 142 +-
hadoop-ozone/interface-client/pom.xml | 4 +-
.../src/main/proto/OMAdminProtocol.proto | 84 ++
.../src/main/proto/OmClientProtocol.proto | 2 +
.../interface-client/src/main/resources/proto.lock | 826 +++++++++++-
hadoop-ozone/interface-storage/pom.xml | 4 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 6 +-
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 5 +-
.../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 2 +-
.../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 36 +-
hadoop-ozone/ozone-manager/pom.xml | 14 +-
.../apache/hadoop/ozone/om/BucketManagerImpl.java | 26 +-
.../hadoop/ozone/om/DirectoryDeletingService.java | 21 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 779 +++++++----
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 14 +
.../apache/hadoop/ozone/om/OMPolicyProvider.java | 5 +-
.../apache/hadoop/ozone/om/OMStarterInterface.java | 2 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 29 +-
.../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 125 ++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 1408 +++++++-------------
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 69 +-
.../hadoop/ozone/om/OzoneManagerStarter.java | 59 +-
.../apache/hadoop/ozone/om/OzoneManagerUtils.java | 180 +++
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 7 +-
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 1 +
.../org/apache/hadoop/ozone/om/VolumeManager.java | 40 +-
.../apache/hadoop/ozone/om/VolumeManagerImpl.java | 300 -----
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 9 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 116 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 185 +--
.../hadoop/ozone/om/request/OMClientRequest.java | 112 +-
.../ozone/om/request/OMKeyRequestFactory.java | 139 ++
.../om/request/bucket/OMBucketCreateRequest.java | 12 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 9 +-
.../om/request/file/OMDirectoryCreateRequest.java | 29 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 8 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 31 +-
.../request/file/OMFileCreateRequestWithFSO.java | 17 +-
.../ozone/om/request/file/OMFileRequest.java | 165 ++-
.../om/request/key/OMAllocateBlockRequest.java | 42 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 10 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 110 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 37 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 52 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 41 +-
.../ozone/om/request/key/OMKeyDeleteRequest.java | 89 +-
.../om/request/key/OMKeyDeleteRequestWithFSO.java | 10 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 32 +-
.../om/request/key/OMKeyRenameRequestWithFSO.java | 14 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 99 +-
.../ozone/om/request/key/OMKeysDeleteRequest.java | 8 +-
.../ozone/om/request/key/OMKeysRenameRequest.java | 9 +-
.../om/request/key/OMPathsPurgeRequestWithFSO.java | 5 +-
.../om/request/key/OMTrashRecoverRequest.java | 2 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 61 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 12 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 5 +-
.../request/key/acl/OMKeyAddAclRequestWithFSO.java | 8 +-
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 4 +-
.../key/acl/OMKeyRemoveAclRequestWithFSO.java | 9 +-
.../om/request/key/acl/OMKeySetAclRequest.java | 4 +-
.../request/key/acl/OMKeySetAclRequestWithFSO.java | 8 +-
.../S3InitiateMultipartUploadRequest.java | 37 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 27 +-
.../multipart/S3MultipartUploadAbortRequest.java | 53 +-
.../S3MultipartUploadAbortRequestWithFSO.java | 10 +-
.../S3MultipartUploadCommitPartRequest.java | 40 +-
.../S3MultipartUploadCommitPartRequestWithFSO.java | 9 +-
.../S3MultipartUploadCompleteRequest.java | 74 +-
.../S3MultipartUploadCompleteRequestWithFSO.java | 26 +-
.../ozone/om/request/upgrade/OMPrepareRequest.java | 9 +-
.../om/request/volume/OMVolumeSetQuotaRequest.java | 5 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 26 +
.../response/file/OMDirectoryCreateResponse.java | 16 +-
.../file/OMDirectoryCreateResponseWithFSO.java | 15 +-
.../om/response/file/OMFileCreateResponse.java | 6 +-
.../response/file/OMFileCreateResponseWithFSO.java | 10 +-
.../response/key/AbstractOMKeyDeleteResponse.java | 16 +-
.../om/response/key/OMAllocateBlockResponse.java | 17 +-
.../key/OMAllocateBlockResponseWithFSO.java | 10 +-
.../ozone/om/response/key/OMKeyCommitResponse.java | 37 +-
.../response/key/OMKeyCommitResponseWithFSO.java | 26 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 17 +-
.../response/key/OMKeyCreateResponseWithFSO.java | 6 +-
.../ozone/om/response/key/OMKeyDeleteResponse.java | 11 +-
.../response/key/OMKeyDeleteResponseWithFSO.java | 14 +-
.../ozone/om/response/key/OMKeyPurgeResponse.java | 3 +-
.../ozone/om/response/key/OMKeyRenameResponse.java | 30 +-
.../response/key/OMKeyRenameResponseWithFSO.java | 23 +-
.../om/response/key/OMKeysDeleteResponse.java | 9 +-
.../om/response/key/OMKeysRenameResponse.java | 11 +-
.../om/response/key/OMOpenKeysDeleteRequest.java | 7 +-
.../om/response/key/OMOpenKeysDeleteResponse.java | 9 +-
.../response/key/OMPathsPurgeResponseWithFSO.java | 14 +-
.../om/response/key/OMTrashRecoverResponse.java | 10 +-
.../ozone/om/response/key/OmKeyResponse.java} | 36 +-
.../om/response/key/acl/OMKeyAclResponse.java | 21 +-
.../response/key/acl/OMKeyAclResponseWithFSO.java | 14 +-
.../S3InitiateMultipartUploadResponse.java | 18 +-
.../S3InitiateMultipartUploadResponseWithFSO.java | 10 +-
.../multipart/S3MultipartUploadAbortResponse.java | 22 +-
.../S3MultipartUploadAbortResponseWithFSO.java | 9 +-
.../S3MultipartUploadCommitPartResponse.java | 18 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 5 +-
.../S3MultipartUploadCompleteResponse.java | 28 +-
.../S3MultipartUploadCompleteResponseWithFSO.java | 11 +-
.../om/snapshot/OzoneManagerSnapshotProvider.java | 7 +
.../ozone/om/upgrade/OMLayoutVersionManager.java | 3 +-
.../protocolPB/OMAdminProtocolServerSideImpl.java | 116 ++
...OzoneManagerProtocolServerSideTranslatorPB.java | 57 +-
.../protocolPB/OzoneManagerRequestHandler.java | 40 +-
.../hadoop/ozone/protocolPB/RequestHandler.java | 4 +-
.../hadoop/ozone/security/AWSV4AuthValidator.java | 10 -
.../security/OzoneBlockTokenSecretManager.java | 5 -
.../hadoop/ozone/security/S3SecurityUtil.java | 82 ++
.../hadoop/ozone/om/TestBucketManagerImpl.java | 63 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 7 +-
.../hadoop/ozone/om/TestOzoneManagerStarter.java | 4 +-
.../ozone/om/request/TestOMRequestUtils.java | 139 +-
.../ozone/om/request/bucket/TestBucketRequest.java | 1 -
.../bucket/TestOMBucketCreateRequestWithFSO.java | 8 +-
.../bucket/TestOMBucketSetPropertyRequest.java | 30 +-
.../request/file/TestOMDirectoryCreateRequest.java | 53 +-
.../file/TestOMDirectoryCreateRequestWithFSO.java | 94 +-
.../om/request/file/TestOMFileCreateRequest.java | 8 +-
.../file/TestOMFileCreateRequestWithFSO.java | 26 +-
.../om/request/key/TestOMAllocateBlockRequest.java | 7 +-
.../key/TestOMAllocateBlockRequestWithFSO.java | 25 +-
.../ozone/om/request/key/TestOMKeyAclRequest.java | 40 +-
.../om/request/key/TestOMKeyAclRequestWithFSO.java | 31 +-
.../om/request/key/TestOMKeyCommitRequest.java | 131 +-
.../request/key/TestOMKeyCommitRequestWithFSO.java | 19 +-
.../om/request/key/TestOMKeyCreateRequest.java | 114 +-
.../request/key/TestOMKeyCreateRequestWithFSO.java | 33 +-
.../om/request/key/TestOMKeyDeleteRequest.java | 26 +-
.../request/key/TestOMKeyDeleteRequestWithFSO.java | 26 +-
.../om/request/key/TestOMKeyRenameRequest.java | 8 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 17 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 4 +-
.../om/request/key/TestOMKeysRenameRequest.java | 20 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 10 +-
.../TestS3InitiateMultipartUploadRequest.java | 30 +-
...estS3InitiateMultipartUploadRequestWithFSO.java | 14 +-
.../s3/multipart/TestS3MultipartRequest.java | 21 +-
.../TestS3MultipartUploadAbortRequest.java | 5 +-
.../TestS3MultipartUploadAbortRequestWithFSO.java | 7 +-
.../TestS3MultipartUploadCommitPartRequest.java | 17 +-
...tS3MultipartUploadCommitPartRequestWithFSO.java | 19 +-
.../TestS3MultipartUploadCompleteRequest.java | 19 +-
...estS3MultipartUploadCompleteRequestWithFSO.java | 28 +-
.../volume/TestOMVolumeSetQuotaRequest.java | 28 +-
.../ozone/om/response/TestCleanupTableInfo.java | 5 +-
.../file/TestOMDirectoryCreateResponse.java | 9 +-
.../file/TestOMDirectoryCreateResponseWithFSO.java | 4 +-
.../file/TestOMFileCreateResponseWithFSO.java | 13 +-
.../response/key/TestOMAllocateBlockResponse.java | 14 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 16 +-
.../om/response/key/TestOMKeyCommitResponse.java | 46 +-
.../key/TestOMKeyCommitResponseWithFSO.java | 18 +-
.../om/response/key/TestOMKeyCreateResponse.java | 12 +-
.../key/TestOMKeyCreateResponseWithFSO.java | 19 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 18 +-
.../key/TestOMKeyDeleteResponseWithFSO.java | 14 +-
.../om/response/key/TestOMKeyRenameResponse.java | 36 +-
.../ozone/om/response/key/TestOMKeyResponse.java | 8 +
.../om/response/key/TestOMKeysDeleteResponse.java | 9 +-
.../om/response/key/TestOMKeysRenameResponse.java | 16 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 18 +-
.../TestS3InitiateMultipartUploadResponse.java | 3 +-
...stS3InitiateMultipartUploadResponseWithFSO.java | 9 +-
.../s3/multipart/TestS3MultipartResponse.java | 28 +-
.../TestS3MultipartUploadAbortResponse.java | 31 +-
.../TestS3MultipartUploadAbortResponseWithFSO.java | 22 +-
...S3MultipartUploadCommitPartResponseWithFSO.java | 15 +-
...stS3MultipartUploadCompleteResponseWithFSO.java | 100 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 21 +-
hadoop-ozone/ozonefs-common/pom.xml | 4 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 18 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 39 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 4 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 67 +
.../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 4 +-
.../org/apache/hadoop/fs/ozone/TestOFSPath.java | 26 +
hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 6 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
hadoop-ozone/pom.xml | 6 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
.../recon/schema/ContainerSchemaDefinition.java | 3 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../apache/hadoop/ozone/recon/ReconHttpServer.java | 5 +-
.../hadoop/ozone/recon/ReconRestServletModule.java | 144 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 26 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 8 -
.../apache/hadoop/ozone/recon/api/AdminOnly.java} | 31 +-
.../hadoop/ozone/recon/api/ContainerEndpoint.java | 10 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 126 +-
.../ozone/recon/api/filters/ReconAdminFilter.java | 113 ++
.../ozone/recon/api/filters/ReconAuthFilter.java | 117 ++
.../ozone/recon/api/filters}/package-info.java | 11 +-
.../hadoop/ozone/recon/api/types/DUResponse.java | 1 +
.../ozone/recon/api/types/ResponseStatus.java | 1 +
.../codec/ContainerReplicaHistoryListCodec.java | 45 +-
.../ozone/recon/fsck/ContainerHealthStatus.java | 17 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 9 +-
.../persistence/ContainerHealthSchemaManager.java | 12 +-
.../ozone/recon/persistence/ContainerHistory.java | 9 +-
.../ozone/recon/scm/ContainerReplicaHistory.java | 27 +-
.../recon/scm/ContainerReplicaHistoryList.java | 32 +-
.../hadoop/ozone/recon/scm/PipelineSyncTask.java | 3 +
.../ozone/recon/scm/ReconContainerManager.java | 21 +-
.../recon/scm/ReconContainerReportHandler.java | 4 +-
.../ozone/recon/scm/ReconDeadNodeHandler.java | 4 +-
.../ReconIncrementalContainerReportHandler.java | 4 +-
.../ozone/recon/scm/ReconPipelineManager.java | 22 +-
.../scm/ReconStorageContainerManagerFacade.java | 9 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 4 +-
.../impl/ReconContainerMetadataManagerImpl.java | 4 +-
.../ozone/recon/spi/impl/ReconDBDefinition.java | 13 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +-
.../ozone/recon/tasks/FileSizeCountTask.java | 8 +-
.../src/views/diskUsage/diskUsage.tsx | 17 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 25 +-
.../ozone/recon/api/TestContainerEndpoint.java | 34 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 7 +-
.../ozone/recon/api/TestNSSummaryEndpoint.java | 15 +-
.../ozone/recon/api/filters/TestAdminFilter.java | 161 +++
.../ozone/recon/fsck/TestContainerHealthTask.java | 22 +-
.../recovery/TestReconOmMetadataManagerImpl.java | 25 +-
.../ozone/recon/scm/TestReconContainerManager.java | 13 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 5 +
.../impl/TestOzoneManagerServiceProviderImpl.java | 17 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 9 +-
.../ozone/recon/tasks/TestFileSizeCountTask.java | 9 +-
.../ozone/recon/tasks/TestNSSummaryTask.java | 16 +-
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 30 +-
hadoop-ozone/s3gateway/pom.xml | 4 +-
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 9 +
.../hadoop/ozone/s3/OzoneClientProducer.java | 73 +-
.../hadoop/ozone/s3/OzoneServiceProvider.java | 28 +-
.../java/org/apache/hadoop/ozone/s3/UgiFilter.java | 169 ---
.../hadoop/ozone/s3/commontypes/CommonPrefix.java | 2 +
.../hadoop/ozone/s3/commontypes/KeyMetadata.java | 1 +
...CommonPrefix.java => ObjectKeyNameAdapter.java} | 37 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 +
.../ozone/s3/endpoint/CopyObjectResponse.java | 2 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 48 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 78 +-
.../hadoop/ozone/s3/endpoint/RootEndpoint.java | 5 +
.../hadoop/ozone/s3/io/S3WrapperInputStream.java | 84 --
.../ozone/s3/signature/AWSSignatureProcessor.java | 46 +-
.../s3/signature/AuthorizationV4QueryParser.java | 5 +-
.../ozone/s3/signature/StringToSignProducer.java | 26 +-
.../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 81 --
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 22 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 31 +-
.../resources/webapps/s3gateway/WEB-INF/web.xml | 8 -
.../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +-
.../org/apache/hadoop/ozone/s3/TestUgiFilter.java | 236 ----
.../TestObjectKeyNameAdapter.java} | 43 +-
.../s3/endpoint/TestAbortMultipartUpload.java | 2 +
.../s3/endpoint/TestInitiateMultipartUpload.java | 2 +
.../hadoop/ozone/s3/endpoint/TestListParts.java | 2 +
.../s3/endpoint/TestMultipartUploadComplete.java | 2 +
.../s3/endpoint/TestMultipartUploadWithCopy.java | 221 ++-
.../hadoop/ozone/s3/endpoint/TestObjectDelete.java | 2 +
.../hadoop/ozone/s3/endpoint/TestObjectGet.java | 2 +
.../hadoop/ozone/s3/endpoint/TestObjectHead.java | 2 +
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 17 +-
.../hadoop/ozone/s3/endpoint/TestPartUpload.java | 2 +
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +
.../hadoop/ozone/s3/util/TestOzoneS3Util.java | 136 --
hadoop-ozone/tools/pom.xml | 4 +-
.../ozone/admin/nssummary/DiskUsageSubCommand.java | 214 +++
.../admin/nssummary/FileSizeDistSubCommand.java | 129 ++
.../ozone/admin/nssummary/NSSummaryAdmin.java | 165 +++
.../ozone/admin/nssummary/NSSummaryCLIUtils.java | 174 +++
.../admin/nssummary/QuotaUsageSubCommand.java | 117 ++
.../ozone/admin/nssummary/SummarySubCommand.java | 115 ++
.../ozone/admin/nssummary}/package-info.java | 15 +-
.../ozone/admin/om/DecommissionOMSubcommand.java | 212 +++
.../ozone/admin/om/FinalizeUpgradeSubCommand.java | 5 +-
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 7 +-
.../admin/scm/FinalizeScmUpgradeSubcommand.java | 5 +-
.../admin/scm/GetScmRatisRolesSubcommand.java | 7 +-
.../hadoop/ozone/debug/DBDefinitionFactory.java | 17 +-
.../org/apache/hadoop/ozone/debug/DBScanner.java | 80 +-
.../apache/hadoop/ozone/debug/DatanodeLayout.java | 110 --
.../apache/hadoop/ozone/debug/PrefixParser.java | 26 +-
.../ozone/debug/container/ContainerCommands.java | 16 +-
.../ozone/freon/ClosedContainerReplicator.java | 10 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 8 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 155 +--
.../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 54 +-
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 908 +++++++++++++
.../containergenerator/GeneratorDatanode.java | 23 +-
.../apache/hadoop/ozone/fsck/ContainerMapper.java | 7 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 2 +-
.../ozone/genesis/BenchMarkOMKeyAllocation.java | 137 --
.../org/apache/hadoop/ozone/genesis/Genesis.java | 2 +-
.../hadoop/ozone/segmentparser/RatisLogParser.java | 4 +-
.../ozone/segmentparser/SCMRatisLogParser.java} | 41 +-
.../org/apache/hadoop/ozone/shell/Handler.java | 17 +
.../ozone/shell/bucket/CreateBucketHandler.java | 24 +-
.../ozone/shell/bucket/ListBucketHandler.java | 10 +-
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 18 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 8 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 18 +-
.../ozone/shell/volume/CreateVolumeHandler.java | 4 +-
.../ozone/shell/volume/ListVolumeHandler.java | 9 +-
.../ozone/debug/TestDBDefinitionFactory.java | 7 +-
pom.xml | 89 +-
800 files changed, 25676 insertions(+), 12630 deletions(-)
create mode 100644 .github/ci.md
rename hadoop-hdds/{framework =>
common}/src/main/java/org/apache/hadoop/hdds/NodeDetails.java (100%)
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ResponseStatus.java
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicatedReplicationConfig.java
(61%)
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTraceAllMethod.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/RefreshVolumeUsageCommandHandler.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorMetrics.java
delete mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/DatanodeMetadataFeatures.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java
delete mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFirstUpgradeLayoutChangeAction.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/VersionedDatanodeFeatures.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScrubberConfiguration.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToScmHA.java
copy hadoop-hdds/docs/themes/ozonedoc/layouts/{index.html =>
shortcodes/image.html} (60%)
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerV2.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerV2.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/AbstractFindTargetGreedy.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceGreedy.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindSourceStrategy.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedyByNetworkTopology.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedyByUsageInfo.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/{PipelineStateManagerV2Impl.java
=> PipelineStateManagerImpl.java} (88%)
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/StateManager.java
delete mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer/TestFindTargetStrategy.java
copy {hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io =>
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/balancer}/package-info.java
(78%)
rename
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/{TestPipelineStateManager.java
=> TestPipelineStateManagerImpl.java} (63%)
delete mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMStoreImplWithOldPipelineIDKeyFormat.java
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/RpcClientTest.java
copy hadoop-ozone/{tools => client}/src/test/resources/log4j.properties (100%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/ExitManager.java =>
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMAdminProtocol.java
(55%)
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OMConfiguration.java
copy
hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/CommonPrefix.java
=> common/src/main/java/org/apache/hadoop/ozone/om/protocol/S3Auth.java} (55%)
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolClientSideImpl.java
copy
hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/CommonPrefix.java
=>
common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OMAdminProtocolPB.java}
(52%)
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/ExitManager.java =>
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerClientProtocol.java
(52%)
delete mode 100755
hadoop-ozone/dist/src/main/compose/upgrade/delete-and-regenerate-data.sh
delete mode 100755 hadoop-ozone/dist/src/main/compose/upgrade/delete-data.sh
delete mode 100755
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/manual-upgrade/0.5.0-1.1.0/callback.sh
rename hadoop-ozone/dist/src/main/compose/{versions =>
upgrade/upgrades/manual-upgrade}/README.md (59%)
rename
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/{1.0.0-1.1.0
=> 1.1.0-1.2.0}/callback.sh (86%)
delete mode 100644 hadoop-ozone/dist/src/main/compose/versions/0.5.0.sh
delete mode 100644 hadoop-ozone/dist/src/main/compose/versions/1.0.0.sh
delete mode 100644 hadoop-ozone/dist/src/main/compose/versions/1.1.0.sh
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithLinks.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterProvider.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStandardOutputUtil.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java
delete mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/TestDatanodeLayoutUpgradeTool.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestAddRemoveOzoneManager.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestBucketOwner.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithBucketLayout.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStore.java
delete mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerBootstrap.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithFailover.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisRequest.java
rename
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/{om/parser/TestOMRatisLogParser.java
=> parser/TestOzoneHARatisLogParser.java} (68%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestNSSummaryAdmin.java
create mode 100644
hadoop-ozone/interface-client/src/main/proto/OMAdminProtocol.proto
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneAclUtils.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerUtils.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMKeyRequestFactory.java
copy
hadoop-ozone/{recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ContainerReplicaHistoryList.java
=>
ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OmKeyResponse.java}
(53%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java
create mode 100644
hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java
copy
hadoop-ozone/{integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java
=> recon/src/main/java/org/apache/hadoop/ozone/recon/api/AdminOnly.java} (56%)
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/filters/ReconAdminFilter.java
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/filters/ReconAuthFilter.java
copy hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io =>
recon/src/main/java/org/apache/hadoop/ozone/recon/api/filters}/package-info.java
(81%)
create mode 100644
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/filters/TestAdminFilter.java
delete mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/UgiFilter.java
copy
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/{CommonPrefix.java
=> ObjectKeyNameAdapter.java} (58%)
delete mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
delete mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/ExitManager.java =>
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
(59%)
delete mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestUgiFilter.java
copy
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/{endpoint/TestObjectDelete.java
=> commontypes/TestObjectKeyNameAdapter.java} (50%)
delete mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/DiskUsageSubCommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/FileSizeDistSubCommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryAdmin.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/NSSummaryCLIUtils.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/QuotaUsageSubCommand.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary/SummarySubCommand.java
rename hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io =>
tools/src/main/java/org/apache/hadoop/ozone/admin/nssummary}/package-info.java
(80%)
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/om/DecommissionOMSubcommand.java
delete mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DatanodeLayout.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/SCMThroughputBenchmark.java
delete mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOMKeyAllocation.java
copy
hadoop-ozone/{s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/CommonPrefix.java
=>
tools/src/main/java/org/apache/hadoop/ozone/segmentparser/SCMRatisLogParser.java}
(53%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
