This is an automated email from the ASF dual-hosted git repository.
siyao pushed a change to branch HDDS-4944
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 35043683a6 HDDS-6371. [Multi-Tenant] Provide OM DB to Apache Ranger
Sync mechanism (#3131)
add cf8eb336ec HDDS-5210. Create GRPC service definition for existing OM
protocol (#2375)
add 3b4e2f58b3 Merge remote-tracking branch 'origin/master' into
HDDS-4440-s3-performance
add 1bce6dfd11 HDDS-5213. Create and a specific OmTransportFactory for
GrpcOmTransport (#2485)
add 46575efe21 HDDS-5211. Start OM Grpc server as part of the OM bootstrap
(#2601)
add a115ebfadc HDDS-5212. Create implementation of OmTransport which uses
Grpc (#2580)
add 1b891fb64f HDDS-5612. Create Ozone s3 authentication token and client
authentication transport (#2655)
add b0f41d67da HDDS-5780. Support s3 authentication on a per request basis
(#2721)
add 1ffbeddcec HDDS 5781. Enable ACLs and support for all s3 file
operations. (#2739)
add c9442d2507 HDDS-5778. Client side unit tests (#2810)
add 4d0bf19895 Commit for merge requested in PR #2945 to merge Apache
Ozone master branch with HDDS-5881 changes INTO HDDS-4400-s3-performance
feature branch. Merge allows s3 gateway gRPC to work with s3 gateway
persistent connection changes made in HDDS-5881 found in master branch.
add abb3637a08 Synchronization added to getOzoneConfiguration for
OzoneClientProducer to satisfy CI findbugs.
add b88a45e994 HDDS-6303. Merge from master to resolve CI workflow issues
& hrpcOmTransport support (#3074)
add 413d4aade2 HDDS-5544. Update GRPC OmTransport implementation for HA
(#2901)
add 0b0c586fec HDDS-5545. Enable TLS for GRPC OmTransport implementation
(#2945)
add 3f585ab05d HDDS-5648. Track CI workflow tests temporarily disabled for
feature branch (#3257)
add d5ade578f4 Revert "HDDS-5648. Track CI workflow tests temporarily
disabled for feature branch (#3257)"
add d87a7461c1 Revert "HDDS-5545. Enable TLS for GRPC OmTransport
implementation (#2945)"
add d57616e165 Revert "HDDS-5544. Update GRPC OmTransport implementation
for HA (#2901)"
add 2522127b1c Revert "HDDS-6303. Merge from master to resolve CI
workflow issues & hrpcOmTransport support (#3074)"
add f7b2dbd8f2 Necessary due to CI workflow issue, HDDS-6239, impacting
feature branch green builds. Merging with Master branch impacts HDDS-4440 PRs
in progress. This merge also requires partial changes that were initially in
PR for HDDS-5545 to be included. Namely, the patch to allow switching between
OmTransports, GrpcOmTransport and Hadoop3OmTransport through ozone
configuration in the OmConfigKeys.
add e3259164ca Trigger Build
add 666525e386 Fix integration test for added configuation field for
selecting OmTransport for s3 gateway - TestOzoneConfigurationFields (added
config key not in xml).
add 0de099dbde Merge remote-tracking branch 'neils-dev/merge-master-s3g'
into HDDS-4440-s3-performance
add 76e238118d HDDS-5544. Update GRPC OmTransport implementation for HA
(#2901)
add 47e457697e HDDS-5545. Enable TLS for GRPC OmTransport implementation
(#2945)
add 062890ee4b HDDS-5648. Track CI workflow tests temporarily disabled for
feature branch (#3257)
add a2e169393c Merge of master into feature branch to sync the feature
branch to the current master. Resolved conflicts affecting feature versioning
PR#3155, propagating s3 exception stack trace HDDS-6257 PR#3066 and
performFailover PR#3160.
add 55d2e06cf2 Fix checkstyle
add 172947815b HDDS-6532. Enable READ audit log by default (#3255)
add 204584b395 HDDS-6515. Intermittent failure in
TestRootedOzoneFileSystem (#3322)
add cec910568c HDDS-5003. Introduce EC replication type (#2064)
add a51efc541b HDDS-4954. Add replicaIndex to the RPC protocols (#2055)
add 23fe4f5e56 Merge remote-tracking branch 'origin/master' into
HDDS-3816-ec
add a915dbfdd5 Merge remote-tracking branch 'origin/master' into
HDDS-3816-ec
add c050b8f67e HDDS-5011. Introduce ECReplicationConfig and Java based
ReplicationConfig implementation (#2068)
add aeedef8c7b Merge branch 'master' into the EC branch
add 1d5c593319 HDDS-5145. Extend Pipline/ReplicationConfig refactor with
ECReplicationConfig (#2206)
add 132911c504 HDDS-5222. ReplicaIndex in Pipeline should be deserialized
in the protobuf message (#2241)
add 9c4008d540 Merge remote-tracking branch 'origin/master' into
HDDS-3816-ec
add ad790b660e HDDS-4883. Persist replicationIndex on datanode side (#2069)
add c960ec7c5a Merge remote-tracking branch 'origin/HDDS-3816-ec' into
HDDS-3816-ec
add f904539c0f Merge remote-tracking branch 'origin/master' into
HDDS-3816-ec
add fbea53d4f5 Merge branch 'master' into EC branch
add b6f960daaa HDDS-5313. ContainerInfo should use ReplicationConfig
(#2320)
add 9e0a2fc4e2 HDDS-5247. EC: Create ECReplicationConfig on client side
based on input string (#2331)
add 5e42c31933 Merge branch 'master' into EC branch.
add 27f57b7d3c Fixed conflict issue as part of merge branch 'master' into
EC branch.
add f53a93e6a9 HDDS-5343 : EC: Add ECReplicationConfig into KeyInfo proto.
(#2340). Contributed by Uma Maheswara Rao G.
add 206c9358a5 HDDS-4892. EC: Implement basic EC pipeline provider (#2353)
add 3a94742765 HDDS-5375. EC: Extend PipelineManager.createPipeline API to
support excluded nodes (#2378)
add 4115827ff9 HDDS-4940 : EC: Implement the ECKeyOutputStream which
should handle the EC mode writes. (#2335)
add cd781b9785 HDDS-5174. EC: Allow EC blocks to be requested from OM
(#2414)
add a0f6881a28 Merge branch 'master' into ec-merge-master
add 0c5c859b99 Fix issues caused by merging master into branch
add 2ef364f47b HDDS-5509. EC: Add missing break in switch statement when
requesting EC blocks (#2468)
add a64945bd7c HDDS-5325. EC: Add configuration to set an EC container
placement policy (#2483)
add 942a23f14e Merge remote-tracking branch 'origin/master' into
merge-master-0803
add f3ce6c7801 HDDS-5510: allocateContainer should handle ec replication
config (#2490)
add 6eba2ff17d HDDS-5543: OMKeyRequest#createFileInfo should handle
ECReplicationConfig (#2501)
add 20851dc199 HDDS-5561. EC: ECReplicationConfig should be immutable
(#2516)
add d2dca9b079 HDDS-5562. EC:
ContainerPlacementPolicyFactory#getPolicyInternal() should not be public.
(#2517)
add 94814e5dba HDDS-5425: Client side pipeline cache key should use host
and port combination (#2510)
add 0e6e00e97e Merge branch 'master' into ec-branch
add db35f8c003 HDDS-4942. EC: Implement ECBlockInputStream to read a
single EC BlockGroup (#2507)
add d6dc278195 EC: Make ECReplicationConfig stored as bucket level
attributes. (#2401)
add 823b505ab5 HDDS-5326. EC: Create a new as many racks as possible
placement policy for EC (#2497)
add 90c4d9286d HDDS-5470 : EC: Add padding and generate parity if the last
stripe is not full (#2455)
add 2df6c9caf5 Merge branch 'master' into ec branch
add e08d5009ab Removed SCMPipelineManager.java which should have been
removed via HDDS-5554, but was somehow missing on an earlier master to ec
branch merge.
add 6b17c7570d HDDS-5745. EC: Pipeline builder should copy replica indexes
from original pipeline (#2642)
add 1d3423de76 HDDS-5477. EC: commit key should consolidate and create one
keyLocationInfo per blockGrp (#2648)
add ebdd1f5e3d HDDS-5682: EC: Provide replication config option from CLI
when creating bucket. (#2639)
add b55107fe76 HDDS-5550. EC: Adapt KeyInputStream to read EC Blocks
(#2653)
add 0a0e62e002 EC: Add Codec and Stripe Size to ECReplicationConfig (#2674)
add fc17518932 HDDS-5784: EC: In BasicRootedOzoneClientAdapterImpl,
Inherit bucket default replication config only in the case of EC. (#2698)
add 5cf251654e HDDS-5741. EC: Remove hard coded chunksize and get from
from ReplicationConfig (#2687)
add 831b3faa27 HDDS-5833: EC: Fix
TestRootedOzoneFileSystem.testBucketDefaultsShouldBeInheritedToFileForEC
failure in branch (#2718)
add 7a67f63429 HDDS-5822 : Writing a large buffer to an EC file duplicates
first chunk in block 1 and 2 (#2714)
add a16ee9d2fb Merge branch 'master' into HDDS-3816-ec
add aee577c171 HDDS-5825: EC: ECKeyOutputStream#close fails if we write
the partial chunk (#2720)
add f478b96360 HDDS-5832. EC: ECKeyOutputStream persists blocks in random
order (#2717)
add d37fe6901b HDDS-5552. EC: Implement seek on ECBlockInputStream (#2723)
add 38e4cca273 HDDS-5755 EC: Refactor ECBlockOutputStreamEntry to
accommodate all block group related ECBlockOuputStreams. (#2702)
add 914ce67af3 HDDS-5364: EC: Adopt EC related utility from Hadoop source
repository (#2733)
add f5dbe00309 Hdds 5491: EC: Write should handle node failures. (#2767)
add c9fa9bdf0f HDDS-5874 EC: Integrate the Codec changes into EC Streams.
(#2777)
add 818118a7ad HDDS-5948: EC: Fix the compile issue in TestOzoneECClient
(Due to concurrent commits) (#2812)
add e82682fe8b HDDS-5551. EC: Implement an Input Stream to reconstruct EC
blocks on demand (#2797)
add f85ac376e6 HDDS-5950. EC: Implement seek on
ECBlockReconstructedStripeInputStream (#2818)
add 438799ba75 HDDS-5953: EC: Review the current flush API and clean up
(#2828)
add 52f05fc349 HDDS-5951. EC: ECBlockReconstructedStripeInputStream should
handle block read failures and continue reading (#2831)
add 67043109c0 Merging master into EC branch
add d5ac6738b7 HDDS-6012: EC: Fix TestOzoneShellHA failures post master
merge with EC branch
add 9a7025a39b HDDS-6035: EC: Fix TestOmMetrics in merge branch
add a0349bd9a9 HDDS-6029: EC: Fix the replication config handling in
OMDirectoryCreateRequest#dirKeyInfoBuilderNoACL (#1)
add 685442c6b8 HDDS-6001. EC: Create ECBlockReconstructedInputStream to
wrap ECBlockReconstructedStripeInputStream (#2848)
add 93fadc9789 HDDS-5979: EC: Change CLI bucket default replication option
name to -type (#2873)
add 6e7d0dc206 HDDS-6009. EC: Optimize
ECBlockReconstructedStripeInputStream where there are no missing data indexes
(#2872)
add 1b9724d60a HDDS-5955: EC: Track the failed servers to add into the
excludeList when invoking allocateBlock (#2849)
add 460aa9b89b HDDS-6020: EC: Provide set replicationConfig option to
bucket (#2880)
add a1d7292bcf HDDS-6010. EC: Create ECBlockInputStreamProxy to choose
between reconstruction and normal reads (#2889)
add 73bddf36d0 Updates to address compile issues after merge
add 3007b5c68d HDDS-6081. EC: Fix ReplicationConfig related test failures
that happened due to merging HDDS-5997 to the EC branch
add c6604833d1 HDDS-5952. EC: ECBlockReconstructedStripeInputStream should
read from blocks in parallel (#2899)
add 86f735418a HDDS-6036: EC: handleStripeFailure should retry (#2910)
add f8fd2ff66b HDDS-6065: EC: Provide CLI option to reset the bucket
replication config (#2927)
add ba192b1de1 HDDS-6132: EC: HandleStripeFailure should not release the
cachebuffers. (#2940)
add 3eb55db014 HDDS-5949. EC: Create reusable buffer pool shared by all EC
input and output streams (#2929)
add 5856411168 HDDS-6179. EC: Provide toString in ECReplicationConfig
(#2986)
add 07b2338525 HDDS-6180. EC: Pipeline creator should ignore creating
pipelines for ZERO factor (#2985)
add aaf6f17045 HDDS-6021: EC: Client side exclude nodes list should expire
after certain time period or based on the list size. (#2973)
add d0467bac1d HDDS-6220. EC: Introduce a gRPC client implementation for
EC with really async WriteChunk and PutBlock. (#3016)
add b40ab3f97b HDDS-6172: EC: Document the Ozone EC (#3006)
add 78ccdd540c HDDS-6131: EC: Replication config from bucket should be
refreshed in o3fs (#3008)
add 6b5c01c843 HDDS-6184. EC: put command should create EC key if bucket
is EC (#2990)
add 4d660c4d79 HDDS-6233. EC: Bucket does not display correct EC
replication details (#3025)
add 2e60a4f8cb Merge master to EC Branch
add 58f8e30538 HDDS-6239. ozonesecure-mr failing with No URLs in
mirrorlist (#3029)
add 78ca097eac HDDS-6083: EC: Fix flakyness of tests around nodefailures
(#3021)
add dfa8141937 HDDS-6237: EC: putBlock should pass close flag true on end
of block group/close file. (#3026)
add ee630e2995 HDDS-6240. EC: Container Info command with json switch
fails for EC containers (#3033)
add 8e52561eab HDDS-6251. EC: Smoketest for ozone admin datanode expects
exactly 3 nodes (#3037)
add ddcfc521af HDDS-6252. EC: Datanode Chunk Validator fails on
encountering EC pipeline (#3035)
add ca6e4c3599 HDDS-6255. EC: Replication Manager should skip EC
Containers (#3038)
add 5d2fa94949 HDDS-6256. EC: Parity blocks are incorrectly padded with
zeros to the chunk size (#3043)
add 9a0f812b3b HDDS-6231. EC: Create EC acceptance test environment and
some basic tests (#3030)
add 722c444aea HDDS-6250. EC: Add replica index to the output in the
container info command (#3041)
add 29c69b109b HDDS-6272: EC: Recon UI failing because of EC replication
factor (#3067)
add 22e34820cf HDDS-6258. EC: Read with stopped but not dead nodes gives
IllegalStateException rather than InsufficientNodesException (#3048)
add c4aec7bb97 HDDS-6259. EC: Pipelines for closed containers should
contain correct replica indexes (#3062)
add dafdb23255 HDDS-6295. EC: Fix unaligned stripe write failure due to
length overflow. (#3076)
add 2d63cc4ec9 HDDS-6194. EC: Freon ockg support EC write (#2996)
add 79f60c4d5c HDDS-6271. EC: Apply fix for HDFS-16422 to the Ozone EC
libraries (#3087)
add aebaff55a2 Merge master to EC branch HDDS-3816-ec
add b2a33b759e HDDS-6320: EC: Fix new checkstyle rule warnings in EC branch
add 4b9d38763a HDDS-6294. EC: Make cluster-wide EC configuration take
effect (#3089)
add 550eadc970 HDDS-6308. EC: Handle Replication Factor to consider EC
Config in Recon UI (#3078)
add 6f64eb77d5 HDDS-6342. EC: Fix large write with multiple stripes upon
stripe failure. (#3108)
add 60a41bc85e HDDS-6249. EC: Fix todo items in TestECKeyOutputStream
(#3102)
add 1d23619164 HDDS-6188: EC: Adapt java side native coder classes from
hadoop. (#3046)
add 387fb1a2c1 HDDS-6232. EC: Update help strings for replication config
(#3113)
add 0a83e7018e HDDS-6341: EC: Fix the race condition in
TestECBlockReconstructedStripeInputStream. (#3111)
add aac07ef790 HDDS-6347. EC: Freon randomKeys EC key support (#3112)
add 6b8d935f73 HDDS-6319. EC: Fix read big file failure with EC policy
10+4. (#3094)
add b9adbcd89a HDDS-6348: EC: PartialStripe failure handling logic is
writing padding bytes also to DNs (#3124)
add b90f00192d HDDS-6372. EC: Do not throw NotImplementedException in
flush() (#3133)
add 78ca327e77 HDDS-6358. EC: Refactor ECKeyOutputStream#write() (#3120)
add 9e3cf98a30 HDDS-6279. EC: Calculate EC replication correctly when
updating bucket usage (#3136)
add e675e3ba3d HDDS-6380. EC: Key Info command should not display legacy
replication fields as they duplicate ReplicationConfig (#3141)
add 9ee6d72f31 HDDS-6373. EC: Exclude pipeline upon container close
instead of exclude DNs. (#3137)
add 38409ea5a0 HDDS-6364. EC: Discard pre-allocated blocks to eliminate
worthless retries. (#3127)
add 9680edf850 HDDS-6384. EC: Ensure EC container usage is updated
correctly when handling reports (#3147)
add 58eac935cd HDDS-6185: EC: EC keys can't be created via S3 interfaces
(#3150)
add 8cff9fd2ad HDDS-6368. EC: Fix broken future chain and cleanup
unnecessary validation function. (#3128)
add 8776430812 HDDS-6382. EC: ListPipelines command should consider EC
Containers (#3161)
add 28482bf2b9 HDDS-6422. EC: Fix too many idle threads during reconstruct
read. (#3168)
add 074203fc0a HDDS-6437. EC: Avoid allocating buffers in EC
Reconstruction Streams until first read (#3181)
add 9675863705 HDDS-6413. EC: OzoneManagerRequestHandler needs to handle
ECReplicationConfig (#3198)
add 06f1eb5ad6 HDDS-6248. EC: Container list command should allow
filtering of EC containers (#3179)
add 03ff3f9592 HDDS-6445. EC: Fix allocateBlock failure due to inaccurate
excludedNodes check. (#3190)
add af48c537a9 HDDS-6452. EC: Fix number of preAllocatedBlocks for EC keys
(#3194)
add 217c1911b2 EC: Check isFullCell inside handleDataWrite (#3202)
add 9cda6a1a06 HDDS-6460. EC: Adjust requested size by EC DataNum in
WritableECContainerProvider (#3203)
add 04c62c53f2 HDDS-6411. EC: OmMultipartKeyInfo needs to handle
ECReplicationConfig (#3210)
add fc6ac0c48c HDDS-6448. EC: Reconstructed Input Streams should free
resources after reading to end of block (#3197)
add f386662a37 HDDS-6381. EC: Improve exception message in
ByteBufferEncodingState (#3143)
add 9ec90f5820 Merge remote-tracking branch 'origin/master' into
HDDS-3816-ec-merge-master
add c55f30edf0 HDDS-6412: EC: Handle ECReplicationConfig in
initiateMultipartUpload API (#3184)
add 16c3e99eae HDDS-6478. EC: Fix ISA-l load hadoop native lib
UnsatisfiedLinkError (#3218)
add 143315c89b HDDS-6409. EC: OzoneMultipartUpload needs to handle
ECReplicationConfig (#3279)
add d9529b32ea HDDS-6434: EC: Overwriting an EC key with a Ratis key fails
(#3274)
add df96b86c8d HDDS-6557. EC: Execute S3 acceptance tests with EC (#3281)
add e9b6215210 Merge master to EC Branch HDDS-3816-ec
add e38607b0cd HDDS-6565. EC: Unify replication-related CLI params (#3293)
add 6c6b740a47 HDDS-6209. EC: [Forward compatibility issue] New client to
older server could fail due to the unavailability for client default
replication config (#3267)
add 70b547be23 HDDS-5909. EC: Onboard EC into upgrade framework (#3262)
add cb0a84b27c Merge master to EC branch HDDS-3816-ec
add 96669856ad Merge EC branch HDDS-3816-ec to master
add 681d0a8f5e HDDS-6471. Only repeat flaky tests (#3247)
add dc9da7d405 HDDS-6606. Rat check failure ignored for
ozone-annotation-processing (#3321)
add be5888b907 HDDS-6580. Introduce KEY_PATH_LOCK under
OzoneManagerLock.Resource (#3308)
add 4345fb1c58 HDDS-6189. Intermittent failure in
TestDirectoryDeletingServiceWithFSO (#3328)
add 03b0a61958 HDDS-6583. Introduce lock strategy pattern implementations
based on configuration flags (#3310)
add cb2754fd9c HDDS-6555. Container Deletion should not depend on
usedBytes being zero (#3276)
add b077cc6123 HDDS-6561. Modify default bucket layout to LEGACY. (#3296)
add 9ac8fbce70 HDDS-6614. EC: Fix Datanode block file INCONSISTENCY during
heavy load. (#3323)
add 2876c54b8d HDDS-6405. Selective checks: run rat for readme change
(#3156)
add 497c7a40b8 HDDS-6617. Skip compile(11) for non-java changes (#3324)
add b8388b096a HDDS-6639. Remove flaky tag for
testOMProxyProviderFailoverOnConnectionFailure (#3343)
add b78adc4c71 HDDS-6620. Gather AuditLog params for S3g (#3325)
add de9fcca1c3 HDDS-6522. Fix
TestOzoneManagerHAWithData#testTwoOMNodesDown (#3341)
add 2ea9e78bf1 Merge remote-tracking branch 'origin/master' into
HDDS-4440-merge
add edec164302 HDDS-6643. Minor fixes for OM gRPC feature (#3347)
add db7f691855 HDDS-6651. Restore S3 Gateway config key check in
TestOzoneConfigurationFields (#3354)
add db778fce96 HDDS-6457. Add freon command to run r/w mix workload using
FS (#3266)
add c6314b987c HDDS-6512. Install Hugo during build if needed (#3287)
add f575e238f8 HDDS-6553. Incorrect timeout when checking balancer
iteration result. (#3272)
add 9d771eb6ed HDDS-6647. Tag some known flaky tests (#3349)
add b4e8e7ac8d HDDS-6635. ReportSubCommand should provide a JSON output
option (#3344)
add 73d4876dfe Revert "HDDS-6579. Add commandline argument for Container
Info command. (#3306)" (#3336)
add f49a691b41 HDDS-6626. Exclude Hadoop Thirdparty from shaded Ozone FS
jars (#3330)
add 4476a5be19 HDDS-6632. ReplicationManagerReport should handle unknown
metric in protobuf (#3335)
add d9a623ad72 HDDS-6598. Add a BackgroundPipelineScrubber to scrub all
pipelines. (#3337)
add 1944b2cda9 HDDS-6624. Make ECBlockReconstructedStripeInputStream
available for datanodes (#3353)
add 240002559e Merge remote-tracking branch 'origin/master' into
HDDS-4440-s3-performance
add 124723a4b2 HDDS-6661. TestBackgroundPipelineScrubber.testRun() fails
intermittently (#3359)
add c251258832 HDDS-6658. BackgroundPipelineCreator does not always stop
quickly (#3357)
add 81b6bbad14 HDDS-6641. datanode usageinfo CLI should provide JSON
output option (#3356)
add 59f60089bb HDDS-6653. Small fixes in PipelineStateMap (#3355)
add d2ac33627a HDDS-6567. Store datanode command queue counts from
heartbeat in DatanodeInfo in SCM (#3329)
add d53cb2c1b7 HDDS-6529. Adding Unit-Test cases for S3-Gateway
Object-Endpoint Metrics (#3302)
add ba9f3425ba HDDS-6684. Move unit tests out of integration (#3375)
add deb88f35ec HDDS-6586: EC: Implement the EC Reconstruction Command with
necessary information (#3345)
add e70bfd0689 Revert "HDDS-5740. Enable ratis by default for SCM.
(#2637)" (#3362)
add 28170f2672 HDDS-6685. Follower OM crashed when validating S3 auth
info. (#3376)
add c33f6a58d8 HDDS-6640. Node.isAncestor might return incorrect result
(#3342)
add c6cd84cc70 HDDS-6377. Redundant loop while doing triggerHeartbeat in
DatanodeStateMachine (#3139)
add 71ce98081d HDDS-6673. Reduce OzoneFS test combinations in ozonesecure
(#3368)
add c99fd20da9 HDDS-6589. Add a new replication manager and change the
existing one to legacy (#3352)
add a3f5021b23 HDDS-6669. Exercise OM in ozonescripts environment (#3367)
add 9ff16ac551 HDDS-6655. Use injected clocks in PipelineManagerImpl and
BackgroundPipelineScrubber to ease testing (#3363)
add f315500c73 HDDS-6551. Introduce StatefulService in scm (#3307)
add 9055a11216 HDDS-6556. Check quota while allocate and update usedBytes
when commit. (#3286)
add a9df285447 HDDS-6563. Revert "HDDS-5867. Update quota when deleting
open keys" (#3282)
add 3889373070 HDDS-6703. Disable failing testPrepareDownedOM until Ratis
upgrade (#3387)
add d9c56aec00 HDDS-6675. Avoid repeating some S3 tests for different
bucket types (#3370)
add 73e69035e0 HDDS-6659: EC: Add BlockGroupLen info as part of PutBlock
in EC Writes for helping in recovery. (#3380)
add e90a44de2f HDDS-6602. Incorrect logic in
PipelineProvider.pickNodesNotUsed (#3318)
add d2c5631d2f HDDS-6716. Adopt RATIS-1560 (#3394)
add 6e0c85eb43 HDDS-6715. Avoid using GrpcClientProtocolService.LOG (#3393)
add d4d8e3c4e9 HDDS-6666. Make Hugo markdown image syntax add responsive
image class (#3364)
add ff351025bc HDDS-6709. Fix bucket usedBytes while versioning is true.
(#3388)
add 1419799bad HDDS-6678. Use secure random in production code (#3374)
add bfc361e1b8 HDDS-6507. Spotbugs transitive dependencies may result in
NoClassDefFound error (#3236)
add a32549cbc1 HDDS-6708. [FSO] Make normalize keyPath on key rename
request (#3386)
add 88b5e854be HDDS-6400. EC: Refactor ECKeyOutputStream for better code
reuse (#3372)
add bccb8c2c83 HDDS-6679. Further splits integration tests (#3373)
add 5459b49538 HDDS-6720. Add a function to get bucket layout info in
Request Validators (#3395)
add 91757d5145 HDDS-6734. ozone admin pipeline list CLI is not backward
compatible (#3406)
add 9f0766ecf1 HDDS-6619. Add freon command to run r/w mix workload using
ObjectStore APIs (#3383)
add 6edcc13900 HDDS-6680. Pre-Finalize behaviour for Bucket Layout
Feature. (#3377)
add 86ffd94735 HDDS-6704. EC: ReplicationManager - create version of
ContainerReplicaCounts applicable to EC (#3405)
add 3b66583c6f HDDS-6560. Add general Grafana dashboard (#3285)
add a6afe188fd HDDS-6630. Explicitly remove node after being chosen (#3332)
add 4046150447 HDDS-6730. Migrate tests in hdds-tools to JUnit5 (#3402)
add f131de5f99 HDDS-5919. In kubernetes OM HA has circular dependency on
service availability (#3185)
add 340f5661e4 HDDS-6731. Migrate simple tests in hdds-server-framework to
JUnit5 (#3412)
add 690b3cd338 HDDS-6748. Intermittent timeout in
TestECBlockReconstructedInputStream#testReadDataWithUnbuffer (#3416)
add ed470082e6 HDDS-6742. Audit operation category mismatch (#3407)
add 71b4feb0ba HDDS-6657. Improve Ozone integrated Ranger configuration
instructions (#3365)
add fb0917328b HDDS-6749. SCM includes itself as peer in addSCM request
(#3413)
add 5ba03daab7 HDDS-6750. Migrate simple tests in hdds-server-scm to
JUnit5 (#3417)
add 44cf924f4d HDDS-6660: EC: Add the DN side Reconstruction Handler
class. (#3399)
add e169d71467 HDDS-6737. Migrate parameterized tests in
hdds-server-framework to JUnit5 (#3414)
add a26ad30f6b HDDS-6596. EC: Support ListBlock from CoordinatorDN (#3410)
add f328aa033b HDDS-6491. Support FSO keys in getExpiredOpenKeys (#3226)
add 1067d353c0 HDDS-6650. S3MultipartUpload support update bucket
usedNamespace. (#3404)
add d5e2b758f0 HDDS-6738. Migrate tests with rules in
hdds-server-framework to JUnit5 (#3415)
add 78a018814c HDDS-6744. EC: ReplicationManager - create
ContainerReplicaPendingOps class and integrate with ContainerManager (#3425)
add d6ece18f28 HDDS-6097. Upgrade to Ratis 2.3.0, gRPC 1.44.0, Netty
4.1.74 (#3433)
add 6cf05906ef HDDS-6777. Upgrade JUnit 5 to 5.8.2 (#3437)
add a6994b2d87 HDDS-6787. Bump gson to 2.8.9 (#3441)
add 2b8c4b7324 HDDS-6779. Register metricSink after initialization in S3
Gateway (#3436)
add fee0de9dd4 HDDS-6784. Missing close() for two RocksDB table iterators
(#3440)
add 51ee649a15 HDDS-6778. Bump Guava to 31.1 (#3435)
add be1180f4b1 HDDS-6786. Bad value for metric NumBytesCommittedCount in
testContainerStateMachineMetrics (#3443)
add ddb1fe24ab HDDS-6756. NPE raised by RpcClient if ServiceInfo
certificates empty (#3428)
add c01fc7907a HDDS-6674. Create compat acceptance split (#3369)
new 79a812af1f Merge remote-tracking branch 'asf/master' into HDDS-4944
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 56 +-
dev-support/annotations/pom.xml | 4 +-
.../org/apache/ozone/annotations/package-info.java | 20 +-
dev-support/ci/selective_ci_checks.bats | 24 +-
dev-support/ci/selective_ci_checks.sh | 24 +-
hadoop-hdds/client/pom.xml | 4 +
.../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 +
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +-
.../hadoop/hdds/scm/XceiverClientManager.java | 13 +-
.../scm/storage/BlockExtendedInputStream.java} | 16 +-
.../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +-
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +-
.../hdds/scm/storage/ECBlockOutputStream.java | 201 ++
.../hdds/scm/storage/ExtendedInputStream.java | 91 +
.../ozone/client/io/BadDataLocationException.java | 56 +
.../ozone/client/io/BlockInputStreamFactory.java | 55 +
.../client/io/BlockInputStreamFactoryImpl.java | 92 +
.../hadoop/ozone/client/io/ECBlockInputStream.java | 416 ++++
.../ozone/client/io/ECBlockInputStreamFactory.java | 58 +
.../client/io/ECBlockInputStreamFactoryImpl.java | 100 +
.../ozone/client/io/ECBlockInputStreamProxy.java | 239 ++
.../client/io/ECBlockReconstructedInputStream.java | 243 ++
.../io/ECBlockReconstructedStripeInputStream.java | 632 +++++
.../client/io/InsufficientLocationsException.java} | 26 +-
.../hadoop/ozone/client/io/package-info.java | 0
.../hadoop/ozone/client/io/ECStreamTestUtil.java | 407 +++
.../client/io/TestBlockInputStreamFactoryImpl.java | 108 +
.../ozone/client/io/TestECBlockInputStream.java | 519 ++++
.../client/io/TestECBlockInputStreamProxy.java | 396 +++
.../io/TestECBlockReconstructedInputStream.java | 377 +++
.../TestECBlockReconstructedStripeInputStream.java | 721 ++++++
hadoop-hdds/common/pom.xml | 4 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 20 +
.../hdds/client/DefaultReplicationConfig.java | 134 +
.../hadoop/hdds/client/ECReplicationConfig.java | 207 ++
.../hadoop/hdds/client/RatisReplicationConfig.java | 7 +
.../hadoop/hdds/client/ReplicationConfig.java | 53 +-
.../hadoop/hdds/client/ReplicationFactor.java | 20 +
.../apache/hadoop/hdds/client/ReplicationType.java | 25 +-
.../hdds/client/StandaloneReplicationConfig.java | 7 +
.../hadoop/hdds/freon/FakeClusterTopology.java | 1 +
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +-
.../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 42 +-
.../hdds/scm/container/ContainerReplicaInfo.java | 14 +-
.../scm/container/ReplicationManagerReport.java | 44 +-
.../scm/container/common/helpers/ExcludeList.java | 48 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 8 +-
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 3 +-
.../org/apache/hadoop/hdds/scm/net/NetUtils.java | 22 +-
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 23 +-
.../java/org/apache/hadoop/hdds/scm/net/Node.java | 28 +
.../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 33 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +-
.../protocol/StorageContainerLocationProtocol.java | 23 +
.../hadoop/hdds/scm/storage/BlockLocationInfo.java | 113 +-
.../hdds/scm/storage/ContainerProtocolCalls.java | 66 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 0
.../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +-
.../org/apache/hadoop/ozone/ClientVersion.java | 3 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../apache/hadoop/ozone/OzoneManagerVersion.java | 2 +
.../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 11 -
.../apache/hadoop/ozone/util/OzoneNetUtils.java | 117 +
.../common/src/main/resources/ozone-default.xml | 111 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 39 +-
.../hdds/client/TestECReplicationConfig.java | 91 +
.../hadoop/hdds/client/TestReplicationConfig.java | 214 +-
.../org/apache/hadoop/hdds/scm/TestSCMHAUtils.java | 54 +
.../hdds/scm/container/TestContainerInfo.java | 86 +
.../scm/container/TestContainerReplicaInfo.java | 31 +
.../container/TestReplicationManagerReport.java | 83 +
.../container/common/helpers/TestExcludeList.java | 67 +
.../hadoop/hdds/scm/container/package-info.java | 1 +
.../hdds/scm/net/TestNetworkTopologyImpl.java | 48 +-
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 +
.../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 +
.../hadoop/ozone/audit/TestOzoneAuditLogger.java | 43 +-
.../apache/hadoop/ozone/ha/TestOzoneNetUtils.java} | 37 +-
.../common/src/test/resources/auditlog.properties | 2 +-
hadoop-hdds/container-service/pom.xml | 2 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 3 +
.../container/common/helpers/ContainerUtils.java | 5 +-
.../container/common/impl/ContainerDataYaml.java | 55 +-
.../common/impl/ContainerLayoutVersion.java | 10 +-
.../container/common/impl/HddsDispatcher.java | 5 +
.../common/report/ContainerReportPublisher.java | 1 +
.../common/report/PipelineReportPublisher.java | 1 +
.../common/statemachine/DatanodeStateMachine.java | 13 +-
.../ReconstructECContainersCommandHandler.java | 78 +
.../states/endpoint/HeartbeatEndpointTask.java | 2 +-
.../server/ratis/ContainerStateMachine.java | 2 +-
.../ECReconstructionCommandInfo.java | 82 +
.../ECReconstructionCoordinatorTask.java | 52 +
.../reconstruction/ECReconstructionSupervisor.java | 72 +
.../container/ec/reconstruction/package-info.java | 34 +-
.../container/keyvalue/KeyValueContainer.java | 1 +
.../container/keyvalue/KeyValueContainerData.java | 10 +
.../ozone/container/keyvalue/KeyValueHandler.java | 15 +
.../container/keyvalue/helpers/ChunkUtils.java | 20 +-
.../keyvalue/impl/FilePerBlockStrategy.java | 2 +-
.../ozoneimpl/ContainerDataScrubberMetrics.java | 1 +
.../commands/ReconstructECContainersCommand.java | 193 ++
.../TestSchemaOneBackwardsCompatibility.java | 3 +-
.../common/impl/TestContainerDataYaml.java | 46 +-
.../states/endpoint/TestHeartbeatEndpointTask.java | 4 +-
.../TestECReconstructionSupervisor.java | 53 +
.../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +-
.../TestReconstructionECContainersCommands.java | 97 +
.../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes
.../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes
.../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes
.../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes
hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++
hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +-
.../docs/content/security/SecurityWithRanger.md | 7 +
.../docs/content/security/SecurityWithRanger.zh.md | 9 +-
hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +-
.../docs/dev-support/bin/make_images_responsive.py | 57 -
hadoop-hdds/docs/pom.xml | 12 +-
.../_markup/render-image.html} | 3 +-
.../client => hadoop-hdds/erasurecode}/pom.xml | 45 +-
.../rawcoder/HadoopNativeECAccessorUtil.java | 62 +
.../io/erasurecode/rawcoder}/package-info.java | 12 +-
.../apache/ozone/erasurecode/BufferAllocator.java | 91 +
.../apache/ozone/erasurecode/CodecRegistry.java | 170 ++
.../java/org/apache/ozone/erasurecode/ECChunk.java | 113 +
.../apache/ozone/erasurecode}/package-info.java | 14 +-
.../rawcoder/AbstractNativeRawDecoder.java | 102 +
.../rawcoder/AbstractNativeRawEncoder.java | 99 +
.../rawcoder/ByteArrayDecodingState.java | 132 +
.../rawcoder/ByteArrayEncodingState.java | 105 +
.../rawcoder/ByteBufferDecodingState.java | 143 ++
.../rawcoder/ByteBufferEncodingState.java | 110 +
.../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++
.../ozone/erasurecode/rawcoder/DecodingState.java | 52 +
.../erasurecode/rawcoder/DummyRawDecoder.java | 45 +
.../erasurecode/rawcoder/DummyRawEncoder.java | 45 +
.../rawcoder/DummyRawErasureCoderFactory.java | 52 +
.../ozone/erasurecode/rawcoder/EncodingState.java | 47 +
.../erasurecode/rawcoder/ErasureCodeNative.java | 96 +
.../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 +
.../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 +
.../rawcoder/NativeRSRawErasureCoderFactory.java | 53 +
.../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 +
.../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 +
.../rawcoder/NativeXORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++
.../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 +
.../rawcoder/RSRawErasureCoderFactory.java | 52 +
.../rawcoder/RawErasureCoderFactory.java | 56 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++
.../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++
.../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 +
.../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 +
.../rawcoder/XORRawErasureCoderFactory.java | 53 +
.../ozone/erasurecode/rawcoder/package-info.java} | 44 +-
.../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 +
.../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 +
.../ozone/erasurecode/rawcoder/util/GF256.java | 333 +++
.../erasurecode/rawcoder/util/GaloisField.java | 565 +++++
.../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++
.../erasurecode/rawcoder/util}/package-info.java | 10 +-
...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 +
.../org/apache/ozone/erasurecode/DumpUtil.java | 96 +
.../ozone/erasurecode/TestCodecRegistry.java | 159 ++
.../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++
.../apache/ozone/erasurecode/package-info.java} | 9 +-
.../rawcoder/RawErasureCoderBenchmark.java | 412 ++++
.../rawcoder/TestCodecRawCoderMapping.java | 68 +
.../erasurecode/rawcoder/TestDummyRawCoder.java | 96 +
.../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 +
.../rawcoder/TestNativeXORRawCoder.java} | 28 +-
.../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +-
.../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 +
.../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++
.../rawcoder/TestRawErasureCoderBenchmark.java} | 31 +-
.../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +-
.../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 +
.../ozone/erasurecode/rawcoder}/package-info.java | 12 +-
hadoop-hdds/framework/pom.xml | 4 +
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 3 +
...lockLocationProtocolClientSideTranslatorPB.java | 8 +
...inerLocationProtocolClientSideTranslatorPB.java | 36 +-
.../SCMBlockLocationFailoverProxyProvider.java | 3 +
.../client/DefaultCertificateClient.java | 9 +-
.../exceptions/TestSCMExceptionResultCodes.java | 19 +-
.../token/TestOzoneBlockTokenIdentifier.java | 16 +-
.../hdds/security/token/TokenVerifierTests.java | 4 +-
.../certificate/authority/TestDefaultCAServer.java | 40 +-
.../certificate/authority/TestDefaultProfile.java | 22 +-
.../client/TestCertificateClientInit.java | 73 +-
.../client/TestDefaultCertificateClient.java | 60 +-
.../x509/certificate/utils/TestCRLCodec.java | 26 +-
.../certificate/utils/TestCertificateCodec.java | 41 +-
.../certificates/TestCertificateSignRequest.java | 60 +-
.../x509/certificates/TestRootCertificate.java | 68 +-
.../security/x509/keys/TestHDDSKeyGenerator.java | 14 +-
.../hdds/security/x509/keys/TestKeyCodec.java | 49 +-
.../apache/hadoop/hdds/server/TestJsonUtils.java | 8 +-
.../apache/hadoop/hdds/server/TestServerUtils.java | 28 +-
.../hadoop/hdds/server/events/TestEventQueue.java | 28 +-
.../hdds/server/events/TestEventQueueChain.java | 2 +-
.../hdds/server/events/TestEventWatcher.java | 46 +-
.../hdds/server/http/TestBaseHttpServer.java | 8 +-
.../hadoop/hdds/server/http/TestHtmlQuoting.java | 31 +-
.../hdds/server/http/TestHttpRequestLog.java | 16 +-
.../server/http/TestHttpRequestLogAppender.java | 12 +-
.../hadoop/hdds/server/http/TestHttpServer2.java | 4 +-
.../hdds/server/http/TestProfileServlet.java | 39 +-
.../server/http/TestPrometheusMetricsSink.java | 47 +-
.../server/http/TestRatisDropwizardExports.java | 13 +-
.../hdds/server/http/TestRatisNameRewrite.java | 65 +-
.../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 33 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 94 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 224 +-
.../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 20 +-
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 129 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 78 +-
.../hadoop/hdds/utils/db/cache/TestTableCache.java | 237 +-
.../src/main/proto/ScmAdminProtocol.proto | 2 +
hadoop-hdds/interface-client/pom.xml | 2 +-
.../src/main/proto/DatanodeClientProtocol.proto | 4 +
.../interface-client/src/main/proto/hdds.proto | 22 +-
hadoop-hdds/interface-server/pom.xml | 2 +-
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 24 +-
.../src/main/proto/ScmServerProtocol.proto | 6 +-
hadoop-hdds/pom.xml | 7 +
hadoop-hdds/server-scm/pom.xml | 2 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 6 +-
.../container/AbstractContainerReportHandler.java | 88 +-
.../hdds/scm/container/ContainerManagerImpl.java | 38 +-
.../hdds/scm/container/ContainerReplica.java | 26 +-
.../hdds/scm/container/ContainerReplicaCount.java | 3 +-
.../scm/container/ECContainerReplicaCount.java | 313 +++
.../scm/container/balancer/ContainerBalancer.java | 135 +-
.../balancer/ContainerBalancerMetrics.java | 70 +-
.../ContainerBalancerSelectionCriteria.java | 2 +-
.../ContainerPlacementPolicyFactory.java | 28 +-
.../algorithms/SCMContainerPlacementMetrics.java | 10 +
.../SCMContainerPlacementRackScatter.java | 329 +++
.../container/replication/ContainerReplicaOp.java | 63 +
.../replication/ContainerReplicaPendingOps.java | 227 ++
...tivityStatusMXBean.java => InflightAction.java} | 27 +-
.../LegacyReplicationManager.java} | 379 +--
.../replication/ReplicationActivityStatus.java | 104 -
.../container/replication/ReplicationManager.java | 453 ++++
.../replication/ReplicationManagerMetrics.java | 1 -
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 +
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 17 +-
.../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 12 +-
.../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 15 +-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 80 +
.../hdds/scm/ha/StatefulServiceStateManager.java | 61 +
.../scm/ha/StatefulServiceStateManagerImpl.java | 124 +
.../hadoop/hdds/scm/metadata/ByteStringCodec.java | 69 +
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hdds/scm/node/CommandQueueReportHandler.java | 49 +
.../hdds/scm/node/DatanodeAdminMonitorImpl.java | 2 +-
.../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 61 +
.../hdds/scm/node/NodeDecommissionManager.java | 2 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 20 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 44 +
.../hadoop/hdds/scm/node/SCMNodeMetrics.java | 16 +
.../scm/pipeline/BackgroundPipelineCreator.java | 31 +-
.../scm/pipeline/BackgroundPipelineScrubber.java | 183 ++
.../hdds/scm/pipeline/ECPipelineProvider.java | 126 +
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 24 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 84 +-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 23 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 5 +
.../scm/pipeline/PipelineStateManagerImpl.java | 20 +
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 44 +-
.../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 20 +
.../scm/pipeline/WritableContainerFactory.java | 8 +
.../scm/pipeline/WritableECContainerProvider.java | 239 ++
.../algorithms/RandomPipelineChoosePolicy.java | 1 +
...lockLocationProtocolServerSideTranslatorPB.java | 17 +-
...inerLocationProtocolServerSideTranslatorPB.java | 255 +-
.../scm/server/OzoneStorageContainerManager.java | 2 +-
.../hdds/scm/server/SCMBlockProtocolServer.java | 4 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 106 +-
.../hadoop/hdds/scm/server/SCMConfigurator.java | 2 +-
.../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 +
.../hdds/scm/server/StorageContainerManager.java | 40 +-
.../scm/server/StorageContainerManagerStarter.java | 3 +
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +-
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 75 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 12 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 48 +-
.../command/TestCommandStatusReportHandler.java | 10 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 27 +
.../hdds/scm/container/SimpleMockNodeManager.java | 20 +
.../container/TestCloseContainerEventHandler.java | 39 +-
.../scm/container/TestContainerActionsHandler.java | 2 +-
.../scm/container/TestContainerManagerImpl.java | 109 +-
.../scm/container/TestContainerReportHandler.java | 207 +-
.../scm/container/TestContainerStateManager.java | 18 +-
.../TestIncrementalContainerReportHandler.java | 34 +-
.../hdds/scm/container/TestReplicationManager.java | 573 +++--
.../scm/container/TestUnknownContainerReport.java | 10 +-
.../container/balancer/TestContainerBalancer.java | 55 +-
.../container/balancer/TestFindTargetStrategy.java | 26 +-
.../algorithms/TestContainerPlacementFactory.java | 43 +-
.../TestContainerPlacementStatusDefault.java | 8 +-
.../TestSCMContainerPlacementCapacity.java | 28 +-
.../TestSCMContainerPlacementRackAware.java | 6 +
...a => TestSCMContainerPlacementRackScatter.java} | 284 +--
.../TestSCMContainerPlacementRandom.java | 36 +-
.../TestContainerReplicaPendingOps.java | 226 ++
.../replication/TestReplicationManagerMetrics.java | 27 +-
.../states/TestContainerReplicaCount.java | 12 +-
.../states/TestECContainerReplicaCount.java | 432 ++++
.../hdds/scm/ha/TestReplicationAnnotation.java | 20 +-
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 10 +-
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 74 +-
.../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 30 +-
.../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 25 +-
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 6 +-
.../hdds/scm/ha/TestSequenceIDGenerator.java | 76 +-
.../ha/TestStatefulServiceStateManagerImpl.java | 89 +
.../hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java | 6 +-
.../hdds/scm/ha/io/TestX509CertificateCodec.java | 13 +-
.../hdds/scm/metadata/TestPipelineIDCodec.java | 6 +-
...ler.java => TestCommandQueueReportHandler.java} | 106 +-
.../hdds/scm/node/TestContainerPlacement.java | 7 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 17 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 38 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 10 +-
.../hdds/scm/node/TestNodeReportHandler.java | 22 +-
.../hadoop/hdds/scm/node/TestNodeStateManager.java | 25 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 37 +-
.../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 28 +-
.../hdds/scm/node/states/TestNodeStateMap.java | 12 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 57 +-
.../pipeline/TestBackgroundPipelineScrubber.java | 109 +
.../hdds/scm/pipeline/TestECPipelineProvider.java | 154 ++
.../scm/pipeline/TestPipelineActionHandler.java | 2 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 290 ++-
.../scm/pipeline/TestPipelineStateManagerImpl.java | 78 +-
.../hdds/scm/pipeline/TestPipelineStateMap.java | 99 +
.../scm/pipeline/TestRatisPipelineProvider.java | 93 +-
.../scm/pipeline/TestSimplePipelineProvider.java | 44 +-
.../pipeline/TestWritableECContainerProvider.java | 446 ++++
.../TestPipelineChoosePolicyFactory.java | 14 +-
.../choose/algorithms/TestLeaderChoosePolicy.java | 26 +-
.../safemode/TestHealthyPipelineSafeModeRule.java | 23 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 21 +-
.../scm/server/TestSCMBlockProtocolServer.java | 22 +-
.../hdds/scm/server/TestSCMContainerMetrics.java | 2 +-
.../server/TestSCMDatanodeHeartbeatDispatcher.java | 61 +-
.../server/TestStorageContainerManagerStarter.java | 18 +-
.../ozone/container/common/TestEndPoint.java | 84 +-
.../placement/TestContainerPlacement.java | 10 +-
.../container/placement/TestDatanodeMetrics.java | 6 +-
.../testutils/ReplicationNodeManagerMock.java | 21 +
.../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 31 +-
.../org/apache/ozone/test/GenericTestUtils.java | 1 +
hadoop-hdds/tools/pom.xml | 7 +
.../hdds/scm/cli/ContainerOperationClient.java | 6 +-
.../hdds/scm/cli/container/InfoSubcommand.java | 10 +-
.../hdds/scm/cli/container/ListSubcommand.java | 28 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 12 +
.../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 162 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 78 +-
.../hdds/scm/cli/container/TestInfoSubCommand.java | 109 +-
.../scm/cli/container/TestReportSubCommand.java | 36 +-
.../datanode/TestContainerBalancerSubCommand.java | 14 +-
.../cli/datanode/TestDecommissionSubCommand.java | 16 +-
.../scm/cli/datanode/TestListInfoSubcommand.java | 14 +-
.../cli/datanode/TestMaintenanceSubCommand.java | 16 +-
.../cli/datanode/TestRecommissionSubCommand.java | 14 +-
.../scm/cli/datanode/TestUsageInfoSubcommand.java | 112 +
.../cli/pipeline/TestListPipelinesSubCommand.java | 238 ++
hadoop-ozone/client/pom.xml | 14 +
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 47 +-
.../org/apache/hadoop/ozone/client/OzoneKey.java | 3 +
.../ozone/client/io/BlockOutputStreamEntry.java | 7 +-
.../client/io/BlockOutputStreamEntryPool.java | 11 +-
.../ozone/client/io/ECBlockOutputStreamEntry.java | 455 ++++
.../client/io/ECBlockOutputStreamEntryPool.java | 89 +
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 625 +++++
.../hadoop/ozone/client/io/KeyInputStream.java | 67 +-
.../hadoop/ozone/client/io/KeyOutputStream.java | 46 +-
.../ozone/client/protocol/ClientProtocol.java | 11 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 228 +-
.../hadoop/ozone/client/MockBlockAllocator.java | 5 +-
.../hadoop/ozone/client/MockDatanodeStorage.java | 29 +-
.../hadoop/ozone/client/MockOmTransport.java | 154 +-
.../ozone/client/MockXceiverClientFactory.java | 47 +-
.../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +-
.../client/MultiNodePipelineBlockAllocator.java | 143 ++
.../ozone/client/SinglePipelineBlockAllocator.java | 30 +-
.../hadoop/ozone/client/TestOzoneClient.java | 49 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 1113 +++++++++
.../ozone/client/io/BlockStreamAccessor.java | 51 +
.../client/io/TestECBlockOutputStreamEntry.java | 128 +
.../ozone/client/io/TestKeyInputStreamEC.java | 115 +
hadoop-ozone/common/pom.xml | 50 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 43 +-
.../hadoop/ozone/om/exceptions/OMException.java | 1 +
.../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 143 ++
.../ozone/om/ha/OMFailoverProxyProvider.java | 22 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 4 +
.../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +-
.../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 212 +-
.../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +-
.../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 +
.../hadoop/ozone/om/lock/OzoneManagerLock.java | 13 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +-
.../ozone/om/protocolPB/GrpcOmTransport.java | 329 +++
.../om/protocolPB/GrpcOmTransportFactory.java} | 24 +-
.../ozone/om/protocolPB/OmTransportFactory.java | 34 +-
...OzoneManagerProtocolClientSideTranslatorPB.java | 60 +-
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 +
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 +
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +-
.../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +-
.../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 +
.../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 325 +++
.../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 222 ++
.../src/test/resources/log4j.properties | 0
hadoop-ozone/csi/pom.xml | 17 +-
hadoop-ozone/dev-support/checks/build.sh | 2 +-
hadoop-ozone/dev-support/checks/integration.sh | 2 +-
hadoop-ozone/dev-support/checks/rat.sh | 15 +-
.../dashboards/Ozone - Overall Metrics.json | 2591 ++++++++++++++++++++
.../dist/src/main/compose/compatibility/test.sh | 2 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/ozone/README.md | 3 +-
.../dist/src/main/compose/ozone/docker-config | 1 +
.../dist/src/main/compose/ozone/prometheus.yml | 11 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 13 +-
.../dist/src/main/compose/ozonescripts/test.sh | 1 +
.../src/main/compose/ozonesecure-ha/docker-config | 1 +
.../src/main/compose/ozonesecure/docker-config | 2 +-
.../dist/src/main/compose/ozonesecure/test.sh | 12 +-
.../compose/upgrade/compose/ha/docker-compose.yaml | 22 +-
.../main/compose/upgrade/compose/ha/docker-config | 3 -
.../src/main/compose/upgrade/compose/ha/load.sh | 2 +-
.../upgrade/compose/non-ha/docker-compose.yaml | 22 +-
.../compose/upgrade/compose/non-ha/docker-config | 2 +-
.../main/compose/upgrade/compose/non-ha/load.sh | 2 +-
hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 10 +-
.../dist/src/main/compose/upgrade/testlib.sh | 8 +
.../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 +
.../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 -
.../dist/src/main/compose/xcompat/docker-config | 2 +
hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 43 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +-
....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 +
.../src/main/smoketest/admincli/container.robot | 4 +-
.../buckethead.robot => basic/single_node.robot} | 22 +-
.../src/main/smoketest/ec/backward-compat.robot | 103 +
.../dist/src/main/smoketest/ec/basic.robot | 64 +
.../dist/src/main/smoketest/ec/lib.resource | 42 +
.../src/main/smoketest/ec/upgrade-ec-check.robot | 45 +
.../dist/src/main/smoketest/ozone-lib/shell.robot | 38 +
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 1 +
.../dist/src/main/smoketest/s3/bucketdelete.robot | 1 +
.../dist/src/main/smoketest/s3/buckethead.robot | 5 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 6 +
.../dist/src/main/smoketest/s3/webui.robot | 1 +
.../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +-
.../dist/src/shell/conf/om-audit-log4j2.properties | 2 +-
.../src/shell/conf/s3g-audit-log4j2.properties | 2 +-
.../src/shell/conf/scm-audit-log4j2.properties | 2 +-
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +-
hadoop-ozone/insight/pom.xml | 2 +-
.../ozone/insight/scm/ReplicaManagerInsight.java | 2 +-
hadoop-ozone/integration-test/pom.xml | 5 +
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +-
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 75 +-
.../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 10 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 237 +-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 13 +-
.../metrics/TestSCMContainerManagerMetrics.java | 2 -
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 -
.../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +-
.../hdds/scm/storage/TestContainerCommandsEC.java | 255 ++
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 -
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +-
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 9 +
.../hadoop/ozone/TestSecureOzoneCluster.java | 2 -
.../hadoop/ozone/TestStorageContainerManager.java | 91 +-
.../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +-
.../rpc/TestContainerReplicationEndToEnd.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 2 -
.../rpc/TestContainerStateMachineFailures.java | 2 -
.../rpc/TestContainerStateMachineFlushDelay.java | 2 -
.../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 1 -
.../client/rpc/TestOzoneRpcClientAbstract.java | 492 ++--
.../client/rpc/read/TestChunkInputStream.java | 7 +-
.../ozone/client/rpc/read/TestInputStreamBase.java | 25 +-
.../ozone/client/rpc/read/TestKeyInputStream.java | 59 +-
.../ozone/container/TestContainerReplication.java | 2 +-
.../apache/hadoop/ozone/container/TestHelper.java | 9 +
.../commandhandler/TestBlockDeletion.java | 7 +-
.../commandhandler/TestDeleteContainerHandler.java | 42 +-
.../ozone/dn/ratis/TestDnRatisLogParser.java | 2 -
.../hadoop/ozone/freon/TestDataValidate.java | 45 +-
.../freon/TestFreonWithDatanodeFastRestart.java | 25 +-
.../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +-
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 326 +++
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 306 +++
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 +-
.../apache/hadoop/ozone/om/OMUpgradeTestUtils.java | 88 +
.../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 272 ++
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 295 ++-
.../hadoop/ozone/om/TestOMUpgradeFinalization.java | 52 +-
.../apache/hadoop/ozone/om/TestObjectStore.java | 19 +-
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +-
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +-
.../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +-
.../ozone/om/TestOzoneManagerHAWithData.java | 1 -
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 8 +
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +-
.../ozone/recon/TestReconWithOzoneManagerHA.java | 2 -
.../hadoop/ozone/scm/TestCloseContainer.java | 2 +-
.../scm/node/TestDecommissionAndMaintenance.java | 4 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 160 +-
hadoop-ozone/interface-client/pom.xml | 30 +-
.../src/main/proto/OmClientProtocol.proto | 44 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 8 +-
hadoop-ozone/ozone-manager/pom.xml | 13 +-
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 140 ++
.../org/apache/hadoop/ozone/om/KeyManager.java | 10 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 15 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 67 +-
.../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +-
.../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 316 ++-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 98 +
.../hadoop/ozone/om/OzoneManagerStarter.java | 3 +
.../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 12 +-
.../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 +
.../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 +
.../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 +
.../ozone/om/lock/RegularBucketLockStrategy.java | 82 +
.../apache/hadoop/ozone/om/lock}/package-info.java | 6 +-
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 7 +-
.../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +-
.../hadoop/ozone/om/request/OMClientRequest.java | 6 +-
.../om/request/bucket/OMBucketCreateRequest.java | 57 +-
.../request/bucket/OMBucketSetPropertyRequest.java | 38 +
.../om/request/file/OMDirectoryCreateRequest.java | 96 +-
.../ozone/om/request/file/OMFileCreateRequest.java | 67 +-
.../request/file/OMFileCreateRequestWithFSO.java | 19 +-
.../om/request/key/OMAllocateBlockRequest.java | 52 +-
.../request/key/OMAllocateBlockRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyCommitRequest.java | 72 +-
.../om/request/key/OMKeyCommitRequestWithFSO.java | 24 +-
.../ozone/om/request/key/OMKeyCreateRequest.java | 62 +-
.../om/request/key/OMKeyCreateRequestWithFSO.java | 19 +-
.../ozone/om/request/key/OMKeyRenameRequest.java | 15 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +-
.../om/request/key/OMOpenKeysDeleteRequest.java | 26 +-
.../S3InitiateMultipartUploadRequest.java | 45 +-
.../S3InitiateMultipartUploadRequestWithFSO.java | 13 +-
.../multipart/S3MultipartUploadAbortRequest.java | 34 +-
.../S3MultipartUploadCommitPartRequest.java | 40 +-
.../S3MultipartUploadCompleteRequest.java | 60 +-
.../om/request/validation/RequestValidations.java | 19 +-
.../om/request/validation/ValidationContext.java | 24 +-
.../response/file/OMFileCreateResponseWithFSO.java | 5 -
.../om/response/key/OMAllocateBlockResponse.java | 14 +-
.../key/OMAllocateBlockResponseWithFSO.java | 10 +-
.../ozone/om/response/key/OMKeyCreateResponse.java | 5 -
.../om/response/key/OMOpenKeysDeleteResponse.java | 13 +-
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 10 +-
.../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +-
...OzoneManagerProtocolServerSideTranslatorPB.java | 14 +-
.../protocolPB/OzoneManagerRequestHandler.java | 231 +-
.../ozone/om/TestGrpcOzoneManagerServer.java | 62 +
.../hadoop/ozone/om/TestOmMetadataManager.java | 107 +-
.../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 +
.../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +-
.../ozone/om/lock/TestOzoneLockProvider.java | 108 +
...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +-
.../ozone/om/request/OMRequestTestUtils.java | 16 -
.../om/request/key/TestOMKeyCreateRequest.java | 81 +-
.../request/key/TestOMKeyRenameRequestWithFSO.java | 99 +
.../ozone/om/request/key/TestOMKeyRequest.java | 31 +-
.../request/key/TestOMOpenKeysDeleteRequest.java | 55 +-
.../s3/multipart/TestS3MultipartRequest.java | 3 +
.../TestS3MultipartUploadCompleteRequest.java | 8 +
.../request/validation/TestRequestValidations.java | 95 +-
.../ozone/om/response/TestCleanupTableInfo.java | 3 +
.../response/key/TestOMAllocateBlockResponse.java | 2 +-
.../key/TestOMAllocateBlockResponseWithFSO.java | 2 +-
.../response/key/TestOMOpenKeysDeleteResponse.java | 59 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 2 +
hadoop-ozone/ozonefs-common/pom.xml | 2 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 45 +-
.../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 +
.../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +
hadoop-ozone/pom.xml | 12 +
hadoop-ozone/recon/pom.xml | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 3 +
.../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +-
.../ozone/recon/api/types/DatanodePipeline.java | 14 +-
.../ozone/recon/api/types/PipelineMetadata.java | 19 +-
.../ozone/recon/scm/ReconContainerManager.java | 7 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 19 +
.../ozone/recon/scm/ReconPipelineManager.java | 5 +-
.../scm/ReconStorageContainerManagerFacade.java | 9 +-
.../src/views/datanodes/datanodes.tsx | 2 +-
.../src/views/pipelines/pipelines.tsx | 2 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
.../scm/AbstractReconContainerManagerTest.java | 8 +-
hadoop-ozone/s3gateway/pom.xml | 26 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 5 +-
.../apache/hadoop/ozone/s3/OzoneClientCache.java | 172 ++
.../hadoop/ozone/s3/OzoneClientProducer.java | 41 +-
.../apache/hadoop/ozone/s3/RequestIdentifier.java | 8 +-
.../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 73 +-
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 58 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 176 +-
.../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 80 +
.../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 87 +-
....hadoop.ozone.om.protocolPB.OmTransportFactory} | 8 +-
.../hadoop/ozone/client/OzoneBucketStub.java | 78 +-
.../ozone/protocolPB/TestGrpcOmTransport.java | 88 +
.../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 25 +-
.../s3/endpoint/TestInitiateMultipartUpload.java | 35 +-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +-
.../ozone/s3/endpoint/TestPermissionCheck.java | 4 +-
.../ozone/s3/metrics/TestS3GatewayMetrics.java | 369 ++-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 ++
hadoop-ozone/tools/pom.xml | 2 +-
.../ozone/freon/AbstractOmBucketReadWriteOps.java | 228 ++
.../hadoop/ozone/freon/BaseFreonGenerator.java | 13 +-
.../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 1 +
.../hadoop/ozone/freon/ContentGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeBlockPutter.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 1 +
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +-
.../freon/FollowerAppendLogEntryGenerator.java | 3 +-
.../java/org/apache/hadoop/ozone/freon/Freon.java | 4 +-
.../ozone/freon/FreonReplicationOptions.java | 73 +
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 1 +
.../ozone/freon/HadoopNestedDirGenerator.java | 1 +
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 3 +-
.../ozone/freon/OmBucketReadWriteFileOps.java | 108 +
.../ozone/freon/OmBucketReadWriteKeyOps.java | 145 ++
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +-
.../ozone/freon/OzoneClientKeyGenerator.java | 23 +-
.../ozone/freon/OzoneClientKeyValidator.java | 2 +
.../hadoop/ozone/freon/RandomKeyGenerator.java | 110 +-
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 1 +
.../hadoop/ozone/freon/SCMThroughputBenchmark.java | 1 +
.../containergenerator/GeneratorDatanode.java | 1 +
.../hadoop/ozone/shell/ReplicationOptions.java | 87 +
.../ozone/shell/ShellReplicationOptions.java} | 23 +-
.../hadoop/ozone/shell/bucket/BucketCommands.java | 1 +
.../ozone/shell/bucket/CreateBucketHandler.java | 31 +-
.../shell/bucket/SetReplicationConfigHandler.java | 54 +
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +-
pom.xml | 87 +-
685 files changed, 40027 insertions(+), 5756 deletions(-)
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ECXceiverClientGrpc.java
copy
hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatusMXBean.java
=>
client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockExtendedInputStream.java}
(68%)
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ECBlockOutputStream.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ExtendedInputStream.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BadDataLocationException.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactory.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockInputStreamFactoryImpl.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStream.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactory.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamFactoryImpl.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockInputStreamProxy.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedInputStream.java
create mode 100644
hadoop-hdds/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockReconstructedStripeInputStream.java
copy
hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java
=>
client/src/main/java/org/apache/hadoop/ozone/client/io/InsufficientLocationsException.java}
(58%)
copy {hadoop-ozone =>
hadoop-hdds}/client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java
(100%)
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/ECStreamTestUtil.java
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestBlockInputStreamFactoryImpl.java
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockInputStream.java
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockInputStreamProxy.java
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockReconstructedInputStream.java
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockReconstructedStripeInputStream.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DefaultReplicationConfig.java
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ECReplicationConfig.java
copy
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java
=>
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockLocationInfo.java
(54%)
rename hadoop-hdds/{framework =>
common}/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java (100%)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/OzoneNetUtils.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestECReplicationConfig.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/TestSCMHAUtils.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerInfo.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/common/helpers/TestExcludeList.java
copy
hadoop-hdds/{server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java
=> common/src/test/java/org/apache/hadoop/ozone/ha/TestOzoneNetUtils.java}
(54%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReconstructECContainersCommandHandler.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCommandInfo.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinatorTask.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionSupervisor.java
copy
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/RequestIdentifier.java
=>
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/package-info.java
(57%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconstructECContainersCommand.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ec/reconstruction/TestECReconstructionSupervisor.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/protocol/commands/TestReconstructionECContainersCommands.java
create mode 100644 hadoop-hdds/docs/content/feature/EC-Chunk-Layout.png
create mode 100644
hadoop-hdds/docs/content/feature/EC-Reads-With-No-Failures.png
create mode 100644
hadoop-hdds/docs/content/feature/EC-Reconstructional-Read.png
create mode 100644
hadoop-hdds/docs/content/feature/EC-Write-Block-Allocation-in-Containers.png
create mode 100644 hadoop-hdds/docs/content/feature/ErasureCoding.md
delete mode 100644 hadoop-hdds/docs/dev-support/bin/make_images_responsive.py
rename hadoop-hdds/docs/themes/ozonedoc/layouts/{shortcodes/image.html =>
_default/_markup/render-image.html} (84%)
copy {hadoop-ozone/client => hadoop-hdds/erasurecode}/pom.xml (57%)
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/HadoopNativeECAccessorUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container =>
erasurecode/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder}/package-info.java
(73%)
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/BufferAllocator.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/CodecRegistry.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/ECChunk.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container =>
erasurecode/src/main/java/org/apache/ozone/erasurecode}/package-info.java (69%)
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawDecoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/AbstractNativeRawEncoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayDecodingState.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteArrayEncodingState.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferDecodingState.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ByteBufferEncodingState.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/CoderUtil.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DecodingState.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawDecoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawEncoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/EncodingState.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/ErasureCodeNative.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawDecoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawEncoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawDecoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawEncoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawDecoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawEncoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RSRawErasureCoderFactory.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderFactory.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureDecoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/RawErasureEncoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawDecoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawEncoder.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/XORRawErasureCoderFactory.java
copy
hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java
=>
erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/package-info.java}
(50%)
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/CodecUtil.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/DumpUtil.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GF256.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/GaloisField.java
create mode 100644
hadoop-hdds/erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util/RSUtil.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container =>
erasurecode/src/main/java/org/apache/ozone/erasurecode/rawcoder/util}/package-info.java
(74%)
create mode 100644
hadoop-hdds/erasurecode/src/main/resources/META-INF/services/org.apache.ozone.erasurecode.rawcoder.RawErasureCoderFactory
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/DumpUtil.java
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCodecRegistry.java
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/TestCoderBase.java
copy
hadoop-hdds/{server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatusMXBean.java
=> erasurecode/src/test/java/org/apache/ozone/erasurecode/package-info.java}
(77%)
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/RawErasureCoderBenchmark.java
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestCodecRawCoderMapping.java
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestDummyRawCoder.java
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeRSRawCoder.java
copy
hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java
=>
erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestNativeXORRawCoder.java}
(57%)
copy
hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java
=>
erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoder.java}
(69%)
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRSRawCoderBase.java
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawCoderBase.java
copy
hadoop-hdds/{framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java
=>
erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestRawErasureCoderBenchmark.java}
(53%)
copy
hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java
=>
erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoder.java}
(75%)
create mode 100644
hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder/TestXORRawCoderBase.java
copy hadoop-hdds/{common/src/test/java/org/apache/hadoop/hdds/scm/container =>
erasurecode/src/test/java/org/apache/ozone/erasurecode/rawcoder}/package-info.java
(83%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ECContainerReplicaCount.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/SCMContainerPlacementRackScatter.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaOp.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerReplicaPendingOps.java
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/{ReplicationActivityStatusMXBean.java
=> InflightAction.java} (58%)
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/{ReplicationManager.java
=> replication/LegacyReplicationManager.java} (88%)
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationActivityStatus.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManager.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManagerImpl.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ByteStringCodec.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueueReportHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineScrubber.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/WritableECContainerProvider.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java
copy
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/{TestSCMContainerPlacementRackAware.java
=> TestSCMContainerPlacementRackScatter.java} (70%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestContainerReplicaPendingOps.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECContainerReplicaCount.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestStatefulServiceStateManagerImpl.java
copy
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/{TestNodeReportHandler.java
=> TestCommandQueueReportHandler.java} (53%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineScrubber.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateMap.java
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestWritableECContainerProvider.java
create mode 100644
hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestUsageInfoSubcommand.java
create mode 100644
hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/pipeline/TestListPipelinesSubCommand.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntry.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntryPool.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MultiNodePipelineBlockAllocator.java
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/BlockStreamAccessor.java
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockOutputStreamEntry.java
create mode 100644
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/io/TestKeyInputStreamEC.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/GrpcOMFailoverProxyProvider.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/QuotaUtil.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
copy
hadoop-ozone/{client/src/test/java/org/apache/hadoop/ozone/client/MockBlockAllocator.java
=>
common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransportFactory.java}
(58%)
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestQuotaUtil.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java
copy hadoop-ozone/{integration-test =>
common}/src/test/resources/log4j.properties (100%)
create mode 100644
hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Overall
Metrics.json
create mode 100755
hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh
create mode 100644
hadoop-ozone/dist/src/main/org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory
copy hadoop-ozone/dist/src/main/smoketest/{s3/buckethead.robot =>
basic/single_node.robot} (58%)
create mode 100644
hadoop-ozone/dist/src/main/smoketest/ec/backward-compat.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/basic.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ec/lib.resource
create mode 100644
hadoop-ozone/dist/src/main/smoketest/ec/upgrade-ec-check.robot
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/OMUpgradeTestUtils.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMBucketLayoutUpgrade.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneConfigUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OBSKeyPathLockStrategy.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockProvider.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneLockStrategy.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock/RegularBucketLockStrategy.java
copy {hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container =>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/lock}/package-info.java
(86%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestGrpcOzoneManagerServer.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneConfigUtil.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/lock/TestOzoneLockProvider.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRenameRequestWithFSO.java
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientCache.java
copy hadoop-ozone/{dev-support/checks/build.sh =>
s3gateway/src/main/resources/META-INF/services/org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory}
(76%)
mode change 100755 => 100644
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/protocolPB/TestGrpcOmTransport.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/AbstractOmBucketReadWriteOps.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonReplicationOptions.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteFileOps.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmBucketReadWriteKeyOps.java
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/ReplicationOptions.java
copy
hadoop-ozone/{client/src/test/java/org/apache/hadoop/ozone/client/MockBlockAllocator.java
=>
tools/src/main/java/org/apache/hadoop/ozone/shell/ShellReplicationOptions.java}
(62%)
create mode 100644
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/SetReplicationConfigHandler.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
