This is an automated email from the ASF dual-hosted git repository.
zanderxu pushed a change to branch HDFS-17384
in repository https://gitbox.apache.org/repos/asf/hadoop.git
omit 141b190c911 HDFS-17506. [FGL] Performance for phase 1 (#6806)
omit 14153f07aab HDFS-17457. [FGL] UTs support fine-grained locking (#6741)
omit 39e0ac08255 HDFS-17459. [FGL] Add documentation (#6737)
omit 43aa4f905ec HDFS-17472. [FGL] gcDeletedSnapshot and getDelegationToken
support FGL (#6743)
omit 02de6df9d44 HDFS-17424. [FGL] DelegationTokenSecretManager supports
fine-grained lock (#6696)
omit 724b9fd5cfd HDFS-17445. [FGL] some operations support fine-grained
locking (#6715)
omit 4f7160853c7 HDFS-17411. [FGL] Client RPCs involving snapshot support
fine-grained lock (#6714)
omit ee0beaee40b HDFS-17389. [FGL] Client RPCs involving read process
supports fine-grained lock (#6590)
omit f0da5122e6d HDFS-17388. [FGL] Client RPCs involving write process
supports fine-grained lock (#6589)
omit 132a63f4ad2 HDFS-17420. [FGL] EditLogTailer and FSEditLogLoader
support FGL (#6679)
omit 771ec5a3df1 HDFS-17412. [FGL] Client RPCs involving maintenance
supports fine-grained lock (#6667)
omit 07c4b73dc50 HDFS-17417. [FGL] HeartbeatManager and
DatanodeAdminMonitor support fine-grained locking (#6656)
omit ea3cbae809a HDFS-17415. [FGL] RPCs in NamenodeProtocol support
fine-grained lock (#6654)
omit a55cd3bb9f4 HDFS-17410. [FGL] Client RPCs that changes file attributes
supports fine-grained lock (#6634)
omit 24139acc865 HDFS-17423. [FGL] BlockManagerSafeMode supports
fine-grained lock (#6645)
omit 1d2b15d9079 HDFS-17416. [FGL] Monitor threads in BlockManager.class
support fine-grained lock (#6647)
omit f64cb049ad0 HDFS-17414. [FGL] RPCs in DatanodeProtocol support
fine-grained lock (#6649)
omit 5c7719b9e60 HDFS-17413. [FGL] CacheReplicationMonitor supports
fine-grained lock (#6641)
omit 19f2c773df4 HDFS-17395. [FGL] Use FSLock to protect
ErasureCodingPolicy related operations (#6579)
omit 0c512c0db00 HDFS-17390. [FGL] FSDirectory supports this fine-grained
locking (#6573)
omit f733d942d10 HDFS-17405. [FGL] Using different metric name to trace
performance for FGL and Global lock (#6600)
omit 03170885c13 HDFS-17398. [FGL] Implement the FGL lock for
FSNLockManager (#6599)
omit ca902d1212e HDFS-17387. [FGL] Abstract the configuration locking mode
(#6572)
omit c4ba19c23b4 HDFS-17394. [FGL] Remove unused WriteHoldCount of
FSNamesystemLock (#6571)
add 782c5014416 HDFS-17451. RBF: fix spotbugs for redundant nullcheck of
dns. (#6697)
add 23286b06321 HDFS-17469. Audit log for reportBadBlocks RPC (#6731)
add 5d0a40c1435 YARN-11672. Create a CgroupHandler implementation for
cgroup v2 (#6734)
add ecf665c6fac YARN-11191. Fix potentional deadlock in GlobalScheduler
refreshQueues (#6732)
add 579b3bcea95 YARN-11690. Update container executor to use
CGROUP2_SUPER_MAGIC in cgroup 2 scenarios (#6771)
add 399299104c5 YARN-11674. Add CPUResourceHandler for cgroup v2. (#6751)
add 027b4c32590 Remove empty queues from the queueByBlockId map (#6772)
add 88ad7db80de HADOOP-19071. Update maven-surefire-plugin from 3.0.0 to
3.2.5. (#6664) Contributed by Shilun Fan.
add daafc8a0b84 HDFS-17367. Add PercentUsed for Different StorageTypes in
JMX (#6735) Contributed by Hualong Zhang.
add 08419c4233d YARN-11675. Update MemoryResourceHandler implementation
for cgroup v2 support (#6760)
add aa169e10937 HADOOP-19159. S3A. Fix documentation of
fs.s3a.committer.abort.pending.uploads (#6778)
add a6f2c4617ec HADOOP-19150: [ABFS] Fixing Test Code for
ITestAbfsRestOperationException#testAuthFailException (#6756)
add 78987a71a64 HADOOP-19151. Support configurable SASL mechanism. (#6740)
add ddb805951ec HDFS-17471. Correct the percentage of sample range.
(#6742). Contributed by fuchaohong.
add 0c9e0b4398a HDFS-17456. Fix the incorrect dfsused statistics of
datanode when appending a file. (#6713). Contributed by fuchaohong.
add 910cb6b887c YARN-11685. Create a config to enable/disable cgroup v2
functionality (#6770)
add a8a58944bd7 HADOOP-19146. S3A: noaa-cors-pds test bucket access with
global endpoint fails (#6723)
add 881034ad452 CachedRecordStore should check if the record state is
expired (#6783)
add 26458984506 HADOOP-19160. hadoop-auth should not depend on
kerb-simplekdc (#6788)
add edf985e2694 HDFS-17500: Add missing operation name while authorizing
some operations (#6776). Contributed by kulkabhay.
add 43e8ca428e2 Revert "HADOOP-18851: Performance improvement for
DelegationTokenSecretManager. (#6001). Contributed by Vikas Kumar."
add 700b3e4800b HDFS-17503. Unreleased volume references because of OOM.
(#6782)
add fb0519253d6 HDFS-17488. DN can fail IBRs with NPE when a volume is
removed (#6759)
add 1d09a64e342 Bump org.bouncycastle:bcprov-jdk18on in /hadoop-project
(#6811)
add b5a90d9500f Bump org.apache.derby:derby in /hadoop-project (#6816)
add ce7d01fac84 YARN-11689. Update the cgroup v2 init error handling
(#6810)
add 12e0ca6b243 HDFS-17522. JournalNode web interfaces lack configs for
X-FRAME-OPTIONS protection (#6814). Contributed by wangzhihui.
add c9270600b7e MAPREDUCE-7474. Improve Manifest committer resilience
(#6716)
add 4cb4d5dd08d HADOOP-19170. Fixes compilation issues on non-Linux
systems (#6822)
add 8d9d58dfc84 HDFS-17099. Fix Null Pointer Exception when stop
namesystem in HDFS.(#6034). Contributed by ConfX.
add 6a4f0be854b HDFS-17514: RBF: Routers should unset cached stateID when
namenode does not set stateID in RPC response header. (#6804)
add bda70450700 HADOOP-19152. Do not hard code security providers. (#6739)
add 39dee8ea191 HADOOP-18958. Improve UserGroupInformation debug log.
(#6255)
add 2e77b7b02ca [HADOOP-18786] Use CDN instead of ASF archive (#5789)
add cab0f4c9ec0 HDFS-17520. [BugFix] TestDFSAdmin.testAllDatanodesReconfig
and TestDFSAdmin.testDecommissionDataNodesReconfig failed (#6812) Contributed
by Zengqiang Xu.
add cf9559eb27f HADOOP-19073 WASB: Fix connection leak in
FolderRenamePending (#6534)
add cfdf1f5e8e0 HADOOP-19172. S3A: upgrade AWS v1 sdk to 1.12.720 (#6823)
add 129d91f7b27 YARN-11692. Support mixed cgroup v1/v2 controller
structure (#6821)
add a97e3022dea HADOOP-19013. Adding
x-amz-server-side-encryption-aws-kms-key-id in the get file attributes for S3A.
(#6646)
add f8dce6c501a HADOOP-18851. Performance improvement for
DelegationTokenSecretManager (#6803)
add 3c00093cb55 HADOOP-19167 Bug Fix: Change of Codec configuration does
not work (#6807)
add 8f92cda35cb HDFS-17509. RBF: Fix ClientProtocol.concat will throw NPE
if tgr is a empty file. (#6784)
add 41eacf4914f MAPREDUCE-7475. Fix non-idempotent unit tests (#6785)
add 47be1ab3b68 HADOOP-18679. Add API for bulk/paged delete of files
(#6726)
add f11a8cfa6ee HADOOP-13147. Constructors must not call overrideable
methods in PureJavaCrc32C (#6408). Contributed by Sebb.
add be28467374b Revert "Bump org.apache.derby:derby in /hadoop-project
(#6816)" (#6841)
add fb156e8f054 HDFS-17464. Improve some logs output in class
FsDatasetImpl (#6724)
add d876505b67c YARN-11681. Update the cgroup documentation with v2
support (#6834)
add d168d3ffeee HADOOP-18325: ABFS: Add correlated metric support for ABFS
operations (#6314)
add f5c5d35eb01 HDFS-17529. RBF: Improve router state store cache entry
deletion (#6833)
add 1baf0e889fe HADOOP-18962. Upgrade kafka to 3.4.0 (#6247)
add 74d30a5dce7 HDFS-17532. RBF: Allow router state store cache update to
overwrite and delete in parallel (#6839)
add f4fde40524d HADOOP-19184. S3A Fix
TestStagingCommitter.testJobCommitFailure (#6843)
add 6c08e8e2aad HADOOP-19156. ZooKeeper based state stores use different
ZK address configs. (#6767). Contributed by liu bin.
add ccb8ff4360d YARN-11687. CGroupV2 resource calculator (#6835)
add d107931fc72 HADOOP-19188. Fix TestHarFileSystem and
TestFilterFileSystem failing after bulk delete API got added. (#6848)
add d00b3acd5ec HADOOP-18679. Followup: change method name case (#6854)
add d8b485a5122 HADOOP-18516: [ABFS][Authentication] Support Fixed SAS
Token for ABFS Authentication (#6552)
add 9f6c997662c YARN-11471. [Federation] FederationStateStoreFacade Cache
Support Caffeine. (#6795) Contributed by Shilun Fan.
add 167d4c8447f YARN-11699. Diagnostics lacks userlimit info when user
capacity has reached its maximum limit (#6849) Contributed by Jiandan Yang.
add f1e2ceb823e HDFS-13603: Do not propagate ExecutionException while
initializing EDEK queues for keys. (#6860)
add f92a8ab8ae5 HADOOP-19190. Skip
ITestS3AEncryptionWithDefaultS3Settings.testEncryptionFileAttributes when
bucket not encrypted with sse-kms (#6859)
add d8d3d538e46 HADOOP-19193. Create orphan commit for website deployment
(#6864)
add 2ee0bf95349 HADOOP-19154. Upgrade bouncycastle to 1.78.1 due to CVEs
(#6755)
add bbb17e76a7a HADOOP-19178: [WASB Deprecation] Updating Documentation on
Upcoming Plans for Hadoop-Azure (#6862)
add 01d257d5aa9 HADOOP-19189. ITestS3ACommitterFactory failing (#6857)
add 10df59e4210 Revert "HADOOP-19071. Update maven-surefire-plugin from
3.0.0 to 3.2.5. (#6664)" (#6875)
add 2e1deee87a5 HADOOP-19137. [ABFS] Prevent ABFS initialization for
non-hierarchal-namespace account if Customer-provided-key configs given. (#6752)
add 776c0a3ab95 HDFS-17539. Make TestFileChecksum fields static (#6853)
add bb30545583c HADOOP-19163. Use hadoop-shaded-protobuf_3_25 (#6858)
add 005030f7a0d HADOOP-18610: [ABFS] OAuth2 Token Provider support for
Azure Workload Identity (#6787)
add 06dd3bfee80 HADOOP-19196. Allow base path to be deleted as well using
Bulk Delete. (#6872)
add 4b1b16a846b HDFS-17551. Fix unit test failure caused by HDFS-17464.
(#6883). Contributed by farmmamba.
add a1f5dc58651 Minor, fix cpu arch compare to use correct Dockerfile
(#6852)
add 2bde5ccb813 HADOOP-19192. Log level is WARN when fail to load native
hadoop libs (#6863)
add 240fddcf17f HADOOP-18931. FileSystem.getFileSystemClass() to log the
jar the .class came from (#6197)
add 2d5fa9e016d HADOOP-18508. S3A: Support parallel integration test runs
on same bucket (#5081)
add 2fbbfe3cc93 HDFS-17546. Implementing HostsFileReader timeout (#6873)
add 90024d8cb13 HDFS-17439. Support -nonSuperUser for
NNThroughputBenchmark: useful for testing auth frameworks such as Ranger (#6677)
add 9710a8d52f9 YARN-11701. [Federation] Enhance Federation Cache Clean
Conditions. (#6889) Contributed by Shilun Fan.
add 1e6411c9ec6 HDFS-17528. FsImageValidation: set txid when saving a new
image (#6828)
add 56c8aa5f1c4 HADOOP-19204. VectorIO regression: empty ranges are now
rejected (#6887)
add 6545b7eeef1 HDFS-17098. DatanodeManager does not handle null storage
type properly. (#6840). Contributed by ConfX.
add 8ac9c1839ac HADOOP-19203. WrappedIO BulkDelete API to raise IOEs as
UncheckedIOExceptions (#6885)
add d3b98cb1b23 HADOOP-19194:Add test to find unshaded dependencies in the
aws sdk (#6865)
add b4ddb2d3bba HDFS-13603: do not propagate ExecutionException and add
maxRetries limit to NameNode edek cache warmup (#6774)
add 134dcf166f5 YARN-11703. Validate accessibility of Node Manager working
directories (#6903)
add c33d8686060 HADOOP-19210. S3A: Speed up some slow unit tests (#6907)
add a57105462be HADOOP-19215. Fix unit tests testSlowConnection and
testBadSetup failed in TestRPC. (#6912). Contributed by farmmamba.
add ae76e9475cd HDFS-17564. EC: Fix the issue of inaccurate metrics when
decommission mark busy DN. (#6911). Contributed by Haiyang Hu.
add 4c55adbb6bc HADOOP-19205. S3A: initialization/close slower than with
v1 SDK (#6892)
add 25e28b41cc3 HADOOP-19216. Upgrade Guice from 4.0 to 5.1.0 to support
Java 17 (#6913). Contributed by Cheng Pan.
add 5a8f70a72eb HDFS-17559. Fix the uuid as null in NameNodeMXBean
(#6906). Contributed by Haiyang Hu.
add 8ca4627a0da HDFS-17557. Fix bug for
TestRedundancyMonitor#testChooseTargetWhenAllDataNodesStop (#6897). Contributed
by Haiyang Hu.
add b1d96f6101c HADOOP-19195. S3A: Upgrade aws sdk v2 to 2.25.53 (#6900)
add 783a8520292 HDFS-17555. Fix NumberFormatException of
NNThroughputBenchmark when configured dfs.blocksize. (#6894). Contributed by
wangzhongwei
add f211af30be9 HDFS-17534. RBF: Support leader follower mode for multiple
subclusters (#6861). Contributed by Yuanbo Liu.
add 8913d379fd2 HDFS-17566. Got wrong sorted block order when StorageType
is considered. (#6934). Contributed by Chenyu Zheng.
add 4f0ee9d67d9 HADOOP-19222. Switch yum repo baseurl due to CentOS 7
sunset (#6932) Contributed by Cheng Pan.
add 51cb858cc8c HADOOP-19208: [ABFS] Fixing logic to determine HNS nature
of account to avoid extra getAcl() calls (#6893)
add 57306566601 HDFS-17574. Make NNThroughputBenchmark support
human-friendly units about blocksize. (#6931). Contributed by wangzhongwei.
add 1360c7574ad HADOOP-19218 Avoid DNS lookup while creating IPC
Connection object (#6916). Contributed by Viraj Jasani.
add ebbe9628d34 HDFS-17558. RBF: Make maxSizeOfFederatedStateToPropagate
work on setResponseHeaderState. (#6902)
add 9a9ad6cc984 YARN-11705. Turn off Node Manager working directories
validation by default (#6948)
add 7638b4727e7 HDFS-17575. SaslDataTransferClient should use
SaslParticipant to create messages. (#6933)
add 9dad697dbca HADOOP-19227. ipc.Server accelerate token negotiation only
for the default mechanism. (#6949)
add a5eb5e96117 HDFS-17576. Support user defined auth Callback. (#6945)
add 1577f57d4c8 HADOOP-19228. ShellCommandFencer#setConfAsEnvVars should
also replace '-' with '_'. (#6936). Contributed by fuchaohong.
add e48cd0e987d Revert "HDFS-17575. SaslDataTransferClient should use
SaslParticipant to create messages. (#6933)"
add b60497ff41e HADOOP-19120. ApacheHttpClient adaptation in ABFS. (#6633)
add e000cbf2771 HADOOP-19218. Addendum. Update TestFSNamesystemLockReport
to exclude hostname resolution from regex. (#6951). Contributed by Viraj Jasani.
add e2a0dca43b5 HDFS-16690. Automatically format unformatted JNs with
JournalNodeSyncer (#6925). Contributed by Aswin M Prabhu.
add 4525c7e35ea HADOOP-19197. S3A: Support AWS KMS Encryption Context
(#6874)
add a5806a9e7bc HADOOP-19161. S3A: option "fs.s3a.performance.flags" to
take list of performance flags (#6789)
add 038636a1b52 HADOOP-19238. Fix create-release script for arm64 based
MacOS (#6962)
add 059e996c02d HDFS-17591. RBF: Router should follow X-FRAME-OPTIONS
protection setting (#6963)
add c593c17255c HADOOP-19237. Upgrade to dnsjava 3.6.1 due to CVEs (#6961)
add 2cf4d638af3 HADOOP-19245. S3ABlockOutputStream no longer sends
progress events in close() (#6974)
add b08d492abd9 HADOOP-19246. Update the yasm rpm download address (#6973)
add 59d5e0bb2e7 HADOOP-19244. Pullout arch-agnostic maven javadoc plugin
configurations in hadoop-common (#6970) Contributed by Cheng Pan.
add b189ef81978 HDFS-17575. SaslDataTransferClient should use
SaslParticipant to create messages. (#6954)
add 2a509117344 HADOOP-17609. Make SM4 support optional for OpenSSL native
code. (#3019)
add 321a6cc55ed HADOOP-19072. S3A: expand optimisations on stores with
"fs.s3a.performance.flags" for mkdir (#6543)
add 74ff00705cf HADOOP-19072. S3A: Override fs.s3a.performance.flags for
tests (ADDENDUM) (#6985)
add fa83c9a8050 HADOOP-19072 S3A: Override fs.s3a.performance.flags for
tests (ADDENDUM 2) (#6993)
add 55a576906dd HADOOP-19131. Assist reflection IO with WrappedOperations
class (#6686)
add fa9bb0d1ac4 HADOOP-19231. Add JacksonUtil to manage Jackson classes
(#6953)
add 5f93edfd707 HADOOP-19153. hadoop-common exports logback as a
transitive dependency (#6999)
add f00094203bf YARN-11709. NodeManager should be shut down or blacklisted
when it cacannot run program /var/lib/yarn-ce/bin/container-executor (#6960)
add bf804cb64be HADOOP-19250. Fix test
TestServiceInterruptHandling.testRegisterAndRaise (#6987)
add b5f88990b72 HADOOP-19136. Upgrade commons-io to 2.16.1. (#6704)
add 59dba6e1bd3 HADOOP-19134. Use StringBuilder instead of StringBuffer.
(#6692). Contributed by PJ Fanning
add e5b76dc99fd HADOOP-19180. EC: Fix calculation errors caused by special
index order (#6813). Contributed by zhengchenyu.
add df08e0de41a HDFS-17605. Reduce memory overhead of TestBPOfferService
(#6996)
add 2fd7cf53fac HADOOP-19253. Google GCS compilation fails due to VectorIO
changes (#7002)
add 33c9ecb6521 HADOOP-19249. KMSClientProvider raises NPE with unauthed
user (#6984)
add b15ed27cfbf HADOOP-19187: [ABFS][FNSOverBlob] AbfsClient Refactoring
to Support Multiple Implementation of Clients. (#6879)
add 012ae9d1aa0 HDFS-17606. Do not require implementing
CustomizedCallbackHandler. (#7005)
add 68fcd7234ca HADOOP-18542. Keep MSI tenant ID and client ID optional
(#4262)
add f6c45e0bcf4 HDFS-17546. Follow-up backport from branch3.3 (#6908)
add 6be04633b55 YARN-11711. Clean Up ServiceScheduler Code. (#6977)
Contributed by Shilun Fan.
add 5745a7dd754 HDFS-16084. Fix getJNIEnv crash due to incorrect state set
to tls var (#6969). Contributed by Kevin Cai.
add 89e38f08ae0 HDFS-17573. Allow turn on both FSImage parallelization and
compression (#6929). Contributed by Sung Dong Kim.
add e4ee3d560bd YARN-10345 HsWebServices containerlogs does not honor ACLs
for completed jobs (#7013)
add 0aab1a29764 HADOOP-19248. Protobuf code generate and replace should
happen together (#6975)
add 0837c84a9f2 Revert "HADOOP-19231. Add JacksonUtil to manage Jackson
classes (#6953)"
add a962aa37e00 HDFS-17599. EC: Fix the mismatch between locations and
indices for mover (#6980)
add b404c8c8f80 HADOOP-19252. Upgrade hadoop-thirdparty to 1.3.0 (#7007)
add 1655acc5e2d HADOOP-19250. [Addendum] Fix test
TestServiceInterruptHandling.testRegisterAndRaise. (#7008)
add 3bbfb2be089 HADOOP-19257. S3A:
ITestAssumeRole.testAssumeRoleBadInnerAuth failure (#7021)
add 94868446104 HADOOP-16928. Make javadoc work on Java 17 (#6976)
add 6c01490f14b YARN-11664. Remove HDFS Binaries/Jars Dependency From Yarn
(#6631)
add 1f302e83fd9 HADOOP-18938. S3A: Fix endpoint region parsing for vpc
endpoints. (#6466)
add 57e62ae07f1 Revert "YARN-11664. Remove HDFS Binaries/Jars Dependency
From Yarn (#6631)"
add a00b1c06f32 HADOOP-19269. Upgrade maven-shade-plugin 3.6.0 (#7029)
add 8c41fbcaf54 Revert "YARN-11709. NodeManager should be shut down or
blacklisted when it ca…" (#7028)
add 6881d12da4b HADOOP-19262: Upgrade wildfly-openssl:1.1.3.Final to
2.1.4.Final to support Java17+ (#7026)
add c9e9bce3610 YARN-11729. Broken 'AM Node Web UI' link on App details
page (#7030)
add c835adb3a8d HADOOP-19201 S3A. Support external-id in assume role
(#6876)
add ea6e0f7cd58 HADOOP-19221. S3A: Unable to recover from failure of
multipart block upload attempt (#6938)
add 4d968add52d HADOOP-19271. NPE in
AbfsManagedApacheHttpConnection.toString() when not connected (#7040)
add 182feb11a0f HADOOP-19277. Files and directories mixed up in
TreeScanResults#dump (#7047)
add d1311e52f78 YARN-11709. NodeManager should be marked unhealthy on
localizer config issues (#7043)
add ee2e5ac4e41 HADOOP-19272. S3A: AWS SDK 2.25.53 warnings logged by
transfer manager (#7048)
add 6bcc2541235 HADOOP-19279. ABFS: Disabling Apache Http Client as
Default Http Client for ABFS Driver(#7055)
add 28538d628ec HADOOP-19164. Hadoop CLI MiniCluster is broken (#7050).
Contributed by Ayush Saxena.
add ff75aa52f3a HDFS-17611. Move all DistCp execution logic to execute()
(#7025)
add 81faae63437 Revert "HDFS-17611. Move all DistCp execution logic to
execute() (#7025)" (#7059)
add d8ca2dbe34b YARN-11730. Mark unreported nodes as LOST on RM Startup/HA
failover (#7049) Contributed by Arjun Mohnot.
add 50fcfd8b298 HDFS-17526. getMetadataInputStream should use
getShareDeleteFileInputStream for windows (#6826). Contributed by Danny Becker
add ae63d8edd54 HDFS-17621. Make PathIsNotEmptyDirectoryException terse
(#7036). Contributed by dzcxzl.
add f5345b881d9 YARN-11560. Fix NPE bug when multi-node enabled with
schedule asynchronously (#6021). Contributed by wangzhongwei.
add fccc268cdea HADOOP-19283. Move all DistCp execution logic to execute()
(#7060)
add 37a74f06921 HADOOP-19285. [ABFS] Restore ETAGS_AVAILABLE to abfs path
capabilities (#7064)
add 68315744f06 HDFS-17040. Namenode web UI should set content type to
application/octet-stream when uploading a file. (#5721)
add b078f86d69c YARN-11733. Fix the order of updating CPU controls with
cgroup v1 (#7069)
add f90a703e480 HADOOP-19165. Drop protobuf 2.5.0 from the distribution
(#7051). Contributed by Ayush Saxena.
add e602c601ddd HADOOP-15760. Upgrade commons-collections to
commons-collections4 (#7006)
add 21ec686be3a YARN-11702: Fix Yarn over allocating containers (#6990)
Contributed by Syed Shameerur Rahman.
add 49a495803a9 HDFS-17381. Distcp of EC files should not be limited to
DFS. (#6551)
add 21cdb450ef2 HADOOP-19284: [ABFS] Allow "fs.azure.account.hns.enabled"
to be set as Account Specific Config (#7062)
add 3d81dde28b6 HDFS-17624. Fix
DFSNetworkTopology#chooseRandomWithStorageType() availableCount when excluded
node is not in selected scope. (#7042). Contributed by fuchaohong.
add 01401d71eff HADOOP-19281. MetricsSystemImpl should not print INFO
message in CLI (#7071)
add 3fda2434196 HADOOP-19290. Operating on / in ChecksumFileSystem throws
NPE. (#7074). Contributed by Ayush Saxena.
add a9b7913d568 HDFS-17626. Reduce lock contention at datanode startup
(#7053). Contributed by Tao Li.
add 9aca73481e0 HADOOP-19280. [ABFS] Initialize client timer only if
metric collection is enabled (#7061)
add 6fd4fea748e HADOOP-19261. Support force close a DomainSocket for
server service (#7057)
add 45b1c86fe51 HADOOP-19294. NPE on maven enforcer with -Pnative on arm
mac (#7082)
add 4ff0dceebda HADOOP-19288. hadoop-client-runtime to exclude dnsjava
InetAddressResolverProvider (#7070)
add e9ed21c065f HADOOP-19286: S3A: Support cross region access when S3
region/endpoint is set (#7067)
add 3f637efaa2c HADOOP-19219. Add JPMS options required by hadoop-common
(#7084) Contributed by Cheng Pan.
add b781882020f YARN-11734. Fix spotbugs in ServiceScheduler#load (#7088)
Contributed by Hualong Zhang.
add 1b5a2a7f65f YARN-11708: Setting maximum-application-lifetime using
AQCv2 templates doesn't apply on the first submitted app (#7041)
add 4e6432a0abf HADOOP-19296. [JDK17] Upgrade maven-war-plugin to 3.4.0.
(#7086) Contributed by Shilun Fan.
add 5ea3a1bd0a5 HADOOP-19286: S3A: Support cross region access when S3
region/endpoint is set (ADDENDUM) (#7098)
add 1f0d9df8875 HDFS-17637. Fix spotbugs in HttpFSFileSystem#getXAttr
(#7099) Contributed by Hualong Zhang.
add 50e6b49e057 HADOOP-19299. HttpReferrerAuditHeader resilience (#7095)
add dc56fc385a4 HADOOP-19295. S3A: large uploads can timeout over slow
links (#7089)
add e4b070025b3 HADOOP-19291. RawLocalFileSystem to allow overlapping
ranges (#7101)
add 9321e322d2a HADOOP-19310. Add JPMS options required by Java 17+
(#7114) Contributed by Cheng Pan.
add 78a08b3b782 MAPREDUCE-7494. File stream leak when LineRecordReader is
interrupted (#7117)
add c63aafd7d1c YARN-11732. Fix potential NPE when calling
SchedulerNode#reservedContainer for CapacityScheduler (#7065). Contributed by
Tao Yang.
add 6589d9f6aae HDFS-17631. Fix RedundantEditLogInputStream.nextOp() state
error when EditLogInputStream.skipUntil() throw IOException (#7066).
Contributed by liuguanghua.
add f931ede86b6 HADOOP-19298. [JDK17] Add a JDK17 profile. (#7085)
Contributed by Shilun Fan.
add 09b348753f8 HDFS-17634. RBF: Fix web UI missing DN last block report
(#7080)
add d1ce9656459 HDFS-17636. Don't add declspec for Windows (#7096)
add 0b3755347c8 HADOOP-19309: S3A: CopyFromLocalFile operation fails when
the source file does not contain file scheme (#7113)
add eb1e30395b3 HDFS-17644:Add log when a node selection is rejected by
BPP UpgradeDomain (#7109)
add 7a7b346b0ab Revert "HADOOP-19298. [JDK17] Add a JDK17 profile. (#7085)
Contributed by Shilun Fan." (#7132)
add 00cddf5bea9 HDFS-17646. Add Option to limit Balancer overUtilized
nodes num in each iteration. (#7120). Contributed by Zhaobo Huang.
add 4f3abd2f482 HDFS-17654. Fix bugs in TestRouterMountTable (#7137).
Contributed by farmmamba.
add e4789a2fd3e HDFS-17607. Reduce the number of times conf is loaded when
DataNode startUp (#7012). Contributed by lei w.
add 66baf1eb51d HADOOP-18682. Move hadoop docker scripts under the main
source code (#6483). Contributed by Christos Bisias.
add df979e70de9 HADOOP-19297. [JDK17] Upgrade maven.plugin-tools.version
to 3.10.2 (#7125) Contributed by Min Yan.
add 9ae01bdbe83 HADOOP-19143. Upgrade commons-cli to 1.9.0 (#7126)
Contributed by Min Yan.
add 51ebc3c20e8 HADOOP-18583. Fix loading of OpenSSL 3.x symbols (#5256)
add f7651e2f63d HADOOP-19243. Upgrade Mockito version to 4.11.0 (#6968)
add 487727a5d10 HADOOP-18960: [ABFS] Making Contract tests run in
sequential and Other Test Fixes (#7104)
add 96572764921 HDFS-17642. Add target node list, exclude source node
list, and exclude target node list parameters to balancer (#7127)
add 9a743bd17f5 HADOOP-19315. Upgrade Apache Avro to 1.11.4 (#7128)
add 2273278d0b0 HADOOP-18708: S3A: Support S3 Client Side Encryption(CSE)
(#6884)
add 7999db55da4 HADOOP-19330. S3A: Add LeakReporter; use in S3AInputStream
(#7151)
add 317db31a9a5 HADOOP-19306. Support user defined auth Callback in
SaslRpcServer. (#7140)
add 7543f3aadab HADOOP-19317. S3A: fs.s3a.connection.expect.continue
controls 100 CONTINUE behavior (#7134)
add c2f13cb7621 HADOOP-18325: [ABFS] Fix metric related test failures due
to missing config (#6847)
add cd2cffe73f9 HADOOP-19342. SaslRpcServer.AuthMethod print INFO messages
in client side. (#7174)
add 31b8b4a8bd5 HADOOP-19336: S3A: Test failures after CSE support added
(#7164)
add b9060fc00df YARN-11738 Modernize SecretManager config (#7144)
add 2ab33c672ea HDFS-17671 Suppress callstack when adding a datanode to
deadnodes (#7180)
add 919bd18c06f HDFS-17641. Add badly distributed blocks metric (#7123)
add 65a5bf3b20c HADOOP-19226: [ABFS][FNSOverBlob] Implementing Azure Rest
APIs on Blob Endpoint for AbfsBlobClient (#6944)
add 964e0897d4a HADOOP-19346. ViewFileSystem.InnerCache: Replaced
ReentrantReadWriteLock with ConcurrentHashMap/putIfAbsent() (#7187)
add 464d7d97f9f HDFS-17668 Treat null SASL negotiated QOP as auth in
DataTransferSasl… (#7171)
add b55c9c7495a HADOOP-19349: S3A : Improve Client Side Encryption
Documentation (#7191)
add 2c2beb715d0 HADOOP-19358. Update command usage of appendToFile. (#7209)
add 86d8fa6d518 HDFS-17679 Use saslClient#hasInitialResponse() instead of
heuristics in SaslParticipant#createFirstMessage() (#7201)
add 2b2acb028c7 HDFS-17682. Fix incorrect command of fs2img tool. (#7206).
Contributed by fuchaohong.
add 93a4dd02cf2 HDFS-17648. Fix BalancerMetrics duplicate registration
issue. (#7133). Contributed by Zhaobo Huang.
add de8b5061d89 HADOOP-19337. Fix ZKFailoverController NPE issue due to
integer overflow in parseInt when initHM. (#7165). Contributed by ConfX.
add 58896f0e135 HADOOP-19339. OutofBounds Exception due to assumption
about buffer size in BlockCompressorStream. Contributed by ConfX.
add aa5fe6f550c YARN-10058. Handle uncaught exception for async-scheduling
threads to prevent scheduler hangs (#7129). Contributed by Tao Yang.
add efb83ecbc34 HADOOP-19360. Disable releases for apache.snapshots repo
(#7218)
add fc42da70cea HADOOP-19357: [ABFS] Optimizations for Retry Handling and
Client Side Throttling (#7216)
add 626455653f5 YARN-11744. Tackle flaky test
testGetRunningContainersToKill (#7227) Contributed by Cheng Pan.
add 5fe4f134afe HADOOP-19366. Install OpenJDK 17 in default ubuntu build
container (#7226) Contributed by Cheng Pan.
add 976fcd914c1 HDFS-17394. [FGL] Remove unused WriteHoldCount of
FSNamesystemLock (#6571)
add c2f88309d56 HDFS-17387. [FGL] Abstract the configuration locking mode
(#6572)
add 1c8216d62f5 HDFS-17398. [FGL] Implement the FGL lock for
FSNLockManager (#6599)
add 58dce1db3ab HDFS-17405. [FGL] Using different metric name to trace
performance for FGL and Global lock (#6600)
add 161ea040a65 HDFS-17390. [FGL] FSDirectory supports this fine-grained
locking (#6573)
add 1d5ffc8f9c3 HDFS-17395. [FGL] Use FSLock to protect
ErasureCodingPolicy related operations (#6579)
add f028f1856c1 HDFS-17413. [FGL] CacheReplicationMonitor supports
fine-grained lock (#6641)
add 1073f0035d5 HDFS-17414. [FGL] RPCs in DatanodeProtocol support
fine-grained lock (#6649)
add c731fc6577b HDFS-17416. [FGL] Monitor threads in BlockManager.class
support fine-grained lock (#6647)
add ccbc34586bc HDFS-17423. [FGL] BlockManagerSafeMode supports
fine-grained lock (#6645)
add 33e82ac9962 HDFS-17410. [FGL] Client RPCs that changes file attributes
supports fine-grained lock (#6634)
add 692a181b648 HDFS-17415. [FGL] RPCs in NamenodeProtocol support
fine-grained lock (#6654)
add 82fcb12d2ff HDFS-17417. [FGL] HeartbeatManager and
DatanodeAdminMonitor support fine-grained locking (#6656)
add da8c178d45c HDFS-17412. [FGL] Client RPCs involving maintenance
supports fine-grained lock (#6667)
add f42c1a6e7b0 HDFS-17420. [FGL] EditLogTailer and FSEditLogLoader
support FGL (#6679)
add 45bc183b942 HDFS-17388. [FGL] Client RPCs involving write process
supports fine-grained lock (#6589)
add 21ba02aee97 HDFS-17389. [FGL] Client RPCs involving read process
supports fine-grained lock (#6590)
add d9adcc19278 HDFS-17411. [FGL] Client RPCs involving snapshot support
fine-grained lock (#6714)
add b2a7c01ef86 HDFS-17445. [FGL] some operations support fine-grained
locking (#6715)
add 5dc73dd3871 HDFS-17424. [FGL] DelegationTokenSecretManager supports
fine-grained lock (#6696)
add 404cb96219c HDFS-17472. [FGL] gcDeletedSnapshot and getDelegationToken
support FGL (#6743)
add 498df817c00 HDFS-17459. [FGL] Add documentation (#6737)
add 147f8332857 HDFS-17457. [FGL] UTs support fine-grained locking (#6741)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (141b190c911)
\
N -- N -- N refs/heads/HDFS-17384 (147f8332857)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/website.yml | 1 +
LICENSE-binary | 44 +-
dev-support/bin/create-release | 4 +-
dev-support/bin/yetus-wrapper | 4 +-
dev-support/docker/Dockerfile_centos_7 | 16 +-
dev-support/docker/Dockerfile_windows_10 | 6 +-
dev-support/docker/pkg-resolver/install-yasm.sh | 2 +-
dev-support/docker/pkg-resolver/packages.json | 6 +-
.../pkg-resolver/set-vault-as-baseurl-centos.sh | 2 +-
.../resources/ensure-jars-have-correct-contents.sh | 2 +
.../hadoop-client-minicluster/pom.xml | 17 +-
.../hadoop-client-runtime/pom.xml | 4 +
.../src/site/markdown/cloud-storage/index.md | 2 +-
.../hadoop-huaweicloud/pom.xml | 18 +-
hadoop-common-project/hadoop-auth/pom.xml | 11 +-
.../dev-support/findbugsExcludeFile.xml | 6 +
hadoop-common-project/hadoop-common/pom.xml | 54 +-
.../hadoop-common/src/main/bin/hadoop-functions.sh | 25 +-
.../java/org/apache/hadoop/conf/Configuration.java | 74 +-
.../java/org/apache/hadoop/crypto/CryptoUtils.java | 77 +
.../apache/hadoop/crypto/JceCtrCryptoCodec.java | 15 +-
.../org/apache/hadoop/crypto/OpensslCipher.java | 16 +
.../hadoop/crypto/OpensslSm4CtrCryptoCodec.java | 4 +
.../org/apache/hadoop/crypto/key/KeyProvider.java | 9 +-
.../org/apache/hadoop/crypto/key/KeyShell.java | 2 +-
.../hadoop/crypto/key/kms/KMSClientProvider.java | 18 +-
.../apache/hadoop/crypto/key/kms/ValueQueue.java | 20 +-
.../org/apache/hadoop/crypto/package-info.java | 12 +-
.../org/apache/hadoop/fs/AbstractFileSystem.java | 2 +-
.../main/java/org/apache/hadoop/fs/BulkDelete.java | 90 ++
.../org/apache/hadoop/fs/BulkDeleteSource.java | 53 +
.../java/org/apache/hadoop/fs/BulkDeleteUtils.java | 66 +
.../org/apache/hadoop/fs/ChecksumFileSystem.java | 4 +-
.../main/java/org/apache/hadoop/fs/ChecksumFs.java | 2 +-
.../org/apache/hadoop/fs/ClosedIOException.java | 22 +-
.../hadoop/fs/CommonConfigurationKeysPublic.java | 19 +
.../apache/hadoop/fs/CommonPathCapabilities.java | 22 +
.../src/main/java/org/apache/hadoop/fs/DF.java | 2 +-
.../org/apache/hadoop/fs/FSDataInputStream.java | 8 +
.../java/org/apache/hadoop/fs/FileContext.java | 32 +-
.../main/java/org/apache/hadoop/fs/FileSystem.java | 72 +-
.../main/java/org/apache/hadoop/fs/FileUtil.java | 23 +-
.../org/apache/hadoop/fs/MultipartUploader.java | 4 +-
.../main/java/org/apache/hadoop/fs/Options.java | 70 +-
.../org/apache/hadoop/fs/RawLocalFileSystem.java | 9 +-
.../java/org/apache/hadoop/fs/RemoteIterator.java | 4 +-
.../org/apache/hadoop/fs/VectoredReadUtils.java | 26 +-
.../org/apache/hadoop/fs/WithErasureCoding.java | 50 +
.../hadoop/fs/impl/AbstractMultipartUploader.java | 2 +-
.../hadoop/fs/impl/DefaultBulkDeleteOperation.java | 97 ++
.../fs/impl/FileSystemMultipartUploader.java | 2 +-
.../java/org/apache/hadoop/fs/impl/FlagSet.java | 327 +++++
.../org/apache/hadoop/fs/impl/LeakReporter.java | 143 ++
.../fs/statistics/FileSystemStatisticNames.java | 26 +-
.../hadoop/fs/statistics/StoreStatisticNames.java | 53 +
.../hadoop/fs/statistics/StreamStatisticNames.java | 8 +
.../hadoop/fs/store/ByteBufferInputStream.java | 199 +++
.../org/apache/hadoop/fs/store/DataBlocks.java | 155 +-
.../fs/store/audit/HttpReferrerAuditHeader.java | 45 +-
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 50 +-
.../java/org/apache/hadoop/ha/HAServiceTarget.java | 2 +-
.../org/apache/hadoop/ha/ShellCommandFencer.java | 10 +-
.../org/apache/hadoop/ha/ZKFailoverController.java | 7 +-
.../java/org/apache/hadoop/io/EnumSetWritable.java | 18 +-
.../java/org/apache/hadoop/io/ObjectWritable.java | 4 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 10 +-
.../hadoop/io/compress/BlockCompressorStream.java | 6 +-
.../org/apache/hadoop/io/compress/CodecPool.java | 4 +
.../io/compress/bzip2/CBZip2InputStream.java | 8 +-
.../io/compress/bzip2/CBZip2OutputStream.java | 82 +-
.../io/compress/zlib/BuiltInZlibDeflater.java | 2 +-
.../io/erasurecode/rawcoder/RSRawDecoder.java | 32 +-
.../org/apache/hadoop/io/file/tfile/Chunk.java | 4 +-
.../org/apache/hadoop/io/wrappedio/WrappedIO.java | 247 ++++
.../hadoop/io/wrappedio/WrappedStatistics.java | 357 +++++
.../hadoop/io/wrappedio/impl/DynamicWrappedIO.java | 500 +++++++
.../wrappedio/impl/DynamicWrappedStatistics.java | 678 +++++++++
.../hadoop/io/wrappedio/impl/package-info.java | 18 +-
.../apache/hadoop/io/wrappedio/package-info.java | 28 +-
.../org/apache/hadoop/ipc/RpcClientException.java | 2 +-
.../java/org/apache/hadoop/ipc/RpcException.java | 2 +-
.../org/apache/hadoop/ipc/RpcServerException.java | 2 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 21 +-
.../hadoop/ipc/UnexpectedServerException.java | 2 +-
.../apache/hadoop/metrics2/impl/MetricsConfig.java | 14 +-
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 19 +-
.../org/apache/hadoop/metrics2/package-info.java | 26 +-
.../main/java/org/apache/hadoop/net/NetUtils.java | 6 +-
.../org/apache/hadoop/net/unix/DomainSocket.java | 71 +-
.../org/apache/hadoop/oncrpc/RpcDeniedReply.java | 2 +-
.../hadoop/security/AccessControlException.java | 6 +-
.../hadoop/security/CustomizedCallbackHandler.java | 121 ++
.../hadoop/security/JniBasedUnixGroupsMapping.java | 2 +-
.../org/apache/hadoop/security/ProviderUtils.java | 2 +-
.../hadoop/security/SaslMechanismFactory.java | 69 +
.../org/apache/hadoop/security/SaslRpcClient.java | 11 +-
.../org/apache/hadoop/security/SaslRpcServer.java | 74 +-
.../hadoop/security/UserGroupInformation.java | 19 +-
.../hadoop/security/alias/CredentialShell.java | 2 +-
.../security/authorize/AuthorizationException.java | 6 +-
.../org/apache/hadoop/security/package-info.java | 14 +-
.../hadoop/security/ssl/SSLHostnameVerifier.java | 8 +-
.../hadoop/security/token/SecretManager.java | 46 +-
.../AbstractDelegationTokenSecretManager.java | 426 +++---
.../delegation/ZKDelegationTokenSecretManager.java | 42 +-
.../java/org/apache/hadoop/util/ClassUtil.java | 22 +-
.../apache/hadoop/util/ConfigurationHelper.java | 126 ++
.../apache/hadoop/util/GenericOptionsParser.java | 2 +-
.../apache/hadoop/util/InstrumentedReadLock.java | 2 +-
.../hadoop/util/InstrumentedReadWriteLock.java | 2 +-
.../apache/hadoop/util/InstrumentedWriteLock.java | 2 +-
.../org/apache/hadoop/util/PureJavaCrc32C.java | 8 +-
.../main/java/org/apache/hadoop/util/Shell.java | 6 +-
.../apache/hadoop/util/ShutdownThreadsHelper.java | 16 +-
.../java/org/apache/hadoop/util/StringUtils.java | 17 +-
.../hadoop/util/curator/ZKCuratorManager.java | 36 +-
.../apache/hadoop/util/dynamic/BindingUtils.java | 214 +++
.../hadoop/util/dynamic/DynConstructors.java | 273 ++++
.../org/apache/hadoop/util/dynamic/DynMethods.java | 544 +++++++
.../apache/hadoop/util/dynamic/package-info.java | 20 +-
.../util/functional/BiFunctionRaisingIOE.java | 16 +
.../hadoop/util/functional/CallableRaisingIOE.java | 19 +
.../util/functional/CommonCallableSupplier.java | 5 +-
.../hadoop/util/functional/FunctionRaisingIOE.java | 15 +
.../hadoop/util/functional/FunctionalIO.java | 89 ++
.../apache/hadoop/util/functional/FutureIO.java | 139 +-
.../util/functional/LazyAtomicReference.java | 152 ++
.../functional/LazyAutoCloseableReference.java | 102 ++
...ableRaisingIOE.java => RunnableRaisingIOE.java} | 21 +-
.../org/apache/hadoop/util/functional/Tuples.java | 104 ++
.../hadoop-common/src/main/native/src/exception.c | 12 +-
.../src/org/apache/hadoop/crypto/OpensslCipher.c | 94 +-
.../apache/hadoop/io/erasurecode/erasure_coder.c | 36 +-
.../apache/hadoop/io/erasurecode/erasure_coder.h | 1 -
.../hadoop/io/erasurecode/erasure_code_test.c | 80 +-
.../src/main/resources/core-default.xml | 64 +
.../src/site/markdown/Benchmarking.md | 2 +
.../src/site/markdown/CLIMiniCluster.md.vm | 6 +-
.../src/site/markdown/FileSystemShell.md | 8 +-
.../src/site/markdown/HadoopDocker.md | 68 +
.../hadoop-common/src/site/markdown/Metrics.md | 11 +
.../src/site/markdown/NativeLibraries.md.vm | 2 +-
.../src/site/markdown/SingleCluster.md.vm | 6 +
.../src/site/markdown/filesystem/bulkdelete.md | 140 ++
.../site/markdown/filesystem/fsdatainputstream.md | 10 +-
.../filesystem/fsdatainputstreambuilder.md | 95 +-
.../filesystem/fsdataoutputstreambuilder.md | 4 +-
.../src/site/markdown/filesystem/index.md | 3 +-
.../apache/hadoop/constants/ConfigConstants.java | 18 +-
.../org/apache/hadoop/constants/package-info.java | 12 +-
.../org/apache/hadoop/crypto/TestCryptoCodec.java | 13 +-
...tCryptoStreamsWithOpensslSm4CtrCryptoCodec.java | 2 +
.../org/apache/hadoop/crypto/TestCryptoUtils.java | 89 ++
.../apache/hadoop/crypto/TestOpensslCipher.java | 10 +
.../apache/hadoop/crypto/key/TestValueQueue.java | 43 +
.../apache/hadoop/fs/FSMainOperationsBaseTest.java | 6 +-
.../hadoop/fs/FileContextCreateMkdirBaseTest.java | 21 +-
.../fs/FileContextMainOperationsBaseTest.java | 8 +-
.../apache/hadoop/fs/TestChecksumFileSystem.java | 7 +
.../fs/TestFSMainOperationsLocalFileSystem.java | 26 +-
.../org/apache/hadoop/fs/TestFilterFileSystem.java | 1 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 2 +
.../test/java/org/apache/hadoop/fs/TestPath.java | 4 +
.../contract/AbstractContractBulkDeleteTest.java | 360 +++++
.../fs/contract/AbstractContractMkdirTest.java | 7 +-
.../AbstractContractMultipartUploaderTest.java | 36 +-
.../contract/AbstractContractVectoredReadTest.java | 49 +-
.../apache/hadoop/fs/contract/ContractOptions.java | 2 +
.../hadoop/fs/contract/ContractTestUtils.java | 22 +-
.../localfs/TestLocalFSContractBulkDelete.java | 10 +-
.../rawlocal/TestRawLocalContractBulkDelete.java | 11 +-
.../org/apache/hadoop/fs/impl/TestFlagSet.java | 431 ++++++
.../apache/hadoop/fs/impl/TestLeakReporter.java | 165 +++
.../hadoop/fs/impl/TestVectoredReadUtils.java | 42 +-
.../java/org/apache/hadoop/fs/shell/TestCount.java | 2 +-
.../TestFSMainOperationsLocalFileSystem.java | 9 -
.../apache/hadoop/ha/TestShellCommandFencer.java | 2 +-
.../test/java/org/apache/hadoop/io/TestText.java | 4 +
.../apache/hadoop/io/compress/TestCodecPool.java | 35 +
.../TestErasureCodingEncodeAndDecode.java | 108 ++
.../io/serializer/avro/TestAvroSerialization.java | 2 +-
.../hadoop/io/wrappedio/impl/TestWrappedIO.java | 484 +++++++
.../io/wrappedio/impl/TestWrappedStatistics.java | 496 +++++++
.../org/apache/hadoop/ipc/MiniRPCBenchmark.java | 12 +-
.../test/java/org/apache/hadoop/ipc/TestIPC.java | 2 +-
.../test/java/org/apache/hadoop/ipc/TestRPC.java | 8 +-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 2 +-
.../java/org/apache/hadoop/ipc/TestServer.java | 2 +
.../hadoop/net/unix/TemporarySocketDirectory.java | 4 +-
.../apache/hadoop/net/unix/TestDomainSocket.java | 61 +-
.../alias/TestCredentialProviderFactory.java | 2 +-
.../security/http/TestCrossOriginFilter.java | 10 +-
.../http/TestRestCsrfPreventionFilter.java | 12 +-
.../launcher/TestServiceInterruptHandling.java | 9 +-
.../java/org/apache/hadoop/test/MockitoUtil.java | 9 +
.../java/org/apache/hadoop/util/TestClassUtil.java | 44 +-
.../hadoop/util/TestConfigurationHelper.java | 174 +++
.../apache/hadoop/util/TestPureJavaCrc32C.java} | 31 +-
.../java/org/apache/hadoop/util/TestShell.java | 2 +-
.../org/apache/hadoop/util/TestStringUtils.java | 9 +
.../util/curator/TestSecureZKCuratorManager.java | 4 +-
.../hadoop/util/curator/TestZKCuratorManager.java | 5 +-
.../apache/hadoop/util/dynamic/Concatenator.java | 85 ++
.../hadoop/util/dynamic/TestDynConstructors.java | 170 +++
.../apache/hadoop/util/dynamic/TestDynMethods.java | 320 +++++
.../hadoop/util/functional/TestFunctionalIO.java | 111 ++
.../hadoop/util/functional/TestLazyReferences.java | 263 ++++
.../util/functional/TestRemoteIterators.java | 2 +-
.../src/test/resources/contract/rawlocal.xml | 5 +
.../src/test/resources/log4j.properties | 4 +-
hadoop-common-project/hadoop-kms/pom.xml | 2 +-
...agerKeyGeneratorKeyProviderCryptoExtension.java | 9 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 2 +-
hadoop-common-project/hadoop-nfs/pom.xml | 2 +-
hadoop-common-project/hadoop-registry/pom.xml | 2 +-
.../hadoop/registry/server/dns/RegistryDNS.java | 2 +-
.../registry/server/dns/TestRegistryDNS.java | 8 +-
hadoop-dist/pom.xml | 94 ++
hadoop-dist/src/main/compose/hadoop/.env | 18 +
hadoop-dist/src/main/compose/hadoop/config | 50 +
.../src/main/compose/hadoop/docker-compose.yaml | 46 +
hadoop-dist/src/main/docker/Dockerfile | 26 +
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 2 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 6 +-
.../java/org/apache/hadoop/hdfs/DFSUtilClient.java | 2 +-
.../apache/hadoop/hdfs/DistributedFileSystem.java | 16 +-
.../hadoop/hdfs/client/DfsPathCapabilities.java | 2 +
.../hdfs/client/impl/BlockReaderLocalLegacy.java | 2 +-
.../client/impl/metrics/BlockReaderIoProvider.java | 2 +-
.../hadoop/hdfs/protocol/ECBlockGroupStats.java | 26 +-
.../hadoop/hdfs/protocol/ReplicatedBlockStats.java | 23 +-
.../datatransfer/sasl/DataTransferSaslUtil.java | 11 +-
.../datatransfer/sasl/SaslDataTransferClient.java | 9 +-
.../datatransfer/sasl/SaslParticipant.java | 28 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 13 +-
.../delegation/DelegationTokenIdentifier.java | 2 +-
.../hdfs/shortcircuit/ShortCircuitCache.java | 2 +-
.../hadoop/hdfs/util/CombinedHostsFileReader.java | 38 +
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 2 +-
.../hadoop-hdfs-native-client/pom.xml | 2 +-
.../src/main/native/libhdfs/CMakeLists.txt | 12 +-
.../src/main/native/libhdfs/jni_helper.c | 19 +-
.../src/main/native/libhdfspp/tests/CMakeLists.txt | 15 +
.../native/libhdfspp/tests/libhdfs_getjni_test.cc | 57 +
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 2 +-
.../server/federation/metrics/FederationMBean.java | 7 +
.../federation/metrics/NamenodeBeanMetrics.java | 17 +
.../hdfs/server/federation/metrics/RBFMetrics.java | 6 +
.../resolver/MembershipNamenodeResolver.java | 2 +
.../MultipleDestinationMountTableResolver.java | 5 +-
.../federation/resolver/NamenodeStatusReport.java | 15 +
.../resolver/order/DestinationOrder.java | 7 +-
.../resolver/order/LeaderFollowerResolver.java | 50 +
.../federation/resolver/order/RandomResolver.java | 2 +-
.../router/NamenodeHeartbeatService.java | 5 +-
.../federation/router/PoolAlignmentContext.java | 7 +-
.../server/federation/router/RBFConfigKeys.java | 5 +
.../federation/router/RouterClientProtocol.java | 68 +-
.../server/federation/router/RouterHttpServer.java | 11 +
.../server/federation/router/RouterRpcServer.java | 48 +-
.../federation/router/RouterStateIdContext.java | 8 +-
.../server/federation/store/CachedRecordStore.java | 21 +-
.../federation/store/driver/StateStoreDriver.java | 89 +-
.../store/driver/StateStoreRecordOperations.java | 25 +
.../store/driver/impl/StateStoreBaseImpl.java | 36 +
.../store/driver/impl/StateStoreFileBaseImpl.java | 1 +
.../store/driver/impl/StateStoreMySQLImpl.java | 1 +
.../store/driver/impl/StateStoreZooKeeperImpl.java | 67 +-
.../federation/store/records/MembershipStats.java | 7 +-
.../records/impl/pb/MembershipStatsPBImpl.java | 14 +-
.../store/records/impl/pb/MountTablePBImpl.java | 4 +
.../hdfs/tools/federation/AddMountAttributes.java | 1 +
.../hadoop/hdfs/tools/federation/RouterAdmin.java | 10 +-
.../src/main/proto/FederationProtocol.proto | 4 +-
.../src/main/resources/hdfs-rbf-default.xml | 17 +
.../src/site/markdown/HDFSRouterFederation.md | 5 +
.../server/federation/metrics/TestRBFMetrics.java | 6 +-
.../resolver/TestMultipleDestinationResolver.java | 17 +
.../resolver/order/TestLeaderFollowerResolver.java | 66 +
.../federation/router/TestObserverWithRouter.java | 103 +-
.../router/TestPoolAlignmentContext.java | 32 +
.../server/federation/router/TestRouterAdmin.java | 21 +-
.../federation/router/TestRouterAdminCLI.java | 37 +-
.../router/TestRouterHeartbeatService.java | 3 +-
.../router/TestRouterHttpServerXFrame.java | 65 +
.../federation/router/TestRouterMountTable.java | 3 +-
.../router/TestRouterMountTableCacheRefresh.java | 3 +-
.../TestRouterMountTableCacheRefreshSecure.java | 3 +-
.../server/federation/router/TestRouterRpc.java | 54 +
.../router/TestRouterRpcMultiDestination.java | 2 +-
.../store/TestStateStoreMembershipState.java | 101 ++
.../federation/store/driver/TestStateStoreZK.java | 24 +-
.../store/records/MockStateStoreDriver.java | 1 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 2 +-
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +
.../apache/hadoop/hdfs/net/DFSNetworkTopology.java | 3 +-
.../datatransfer/sasl/SaslDataTransferServer.java | 30 +-
.../protocol/datatransfer/sasl/package-info.java | 14 +-
.../qjournal/protocol/InterQJournalProtocol.java | 10 +
...nterQJournalProtocolServerSideTranslatorPB.java | 16 +
.../InterQJournalProtocolTranslatorPB.java | 14 +
.../qjournal/server/JournalNodeHttpServer.java | 10 +
.../hdfs/qjournal/server/JournalNodeRpcServer.java | 28 +-
.../hdfs/qjournal/server/JournalNodeSyncer.java | 81 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 93 +-
.../hdfs/server/balancer/BalancerParameters.java | 70 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 8 +-
.../hdfs/server/blockmanagement/BlockManager.java | 16 +-
.../BlockPlacementPolicyDefault.java | 7 +-
.../BlockPlacementPolicyWithUpgradeDomain.java | 3 +
.../blockmanagement/BlockReconstructionWork.java | 2 +-
.../blockmanagement/CombinedHostFileManager.java | 11 +-
.../server/blockmanagement/DatanodeDescriptor.java | 2 +-
.../server/blockmanagement/DatanodeManager.java | 10 +-
.../server/blockmanagement/ErasureCodingWork.java | 8 +-
.../blockmanagement/LowRedundancyBlocks.java | 20 +-
.../blockmanagement/PendingDataNodeMessages.java | 17 +-
.../server/blockmanagement/ReplicationWork.java | 3 +-
.../server/blockmanagement/StorageTypeStats.java | 19 +
.../hdfs/server/datanode/BPOfferService.java | 7 +
.../hdfs/server/datanode/BPServiceActor.java | 2 +-
.../hadoop/hdfs/server/datanode/BlockSender.java | 4 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 3 +-
.../server/datanode/DataNodeFaultInjector.java | 6 +
.../hadoop/hdfs/server/datanode/DataStorage.java | 4 +-
.../hdfs/server/datanode/DirectoryScanner.java | 4 +-
.../hadoop/hdfs/server/datanode/LocalReplica.java | 6 +
.../server/datanode/ProfilingFileIoEvents.java | 2 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 26 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 2 +
.../fsdataset/impl/ReplicaCachingGetSpaceUsed.java | 2 +-
.../server/datanode/metrics/DataNodeMetrics.java | 5 +
.../org/apache/hadoop/hdfs/server/mover/Mover.java | 18 +-
.../server/namenode/EncryptionZoneManager.java | 2 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 50 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 13 +-
.../hdfs/server/namenode/FSImageFormatPBINode.java | 2 +
.../server/namenode/FSImageFormatProtobuf.java | 52 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 84 +-
.../hdfs/server/namenode/FsImageValidation.java | 7 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 5 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 2 +-
.../hdfs/server/namenode/NameNodeMXBean.java | 8 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 5 +-
.../namenode/RedundantEditLogInputStream.java | 2 +
.../namenode/metrics/ReplicatedBlocksMBean.java | 5 +
.../hdfs/server/namenode/snapshot/DiffList.java | 2 +-
.../namenode/snapshot/FSImageFormatPBSnapshot.java | 4 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 17 +-
.../java/org/apache/hadoop/hdfs/tools/DFSck.java | 2 +-
.../offlineEditsViewer/OfflineEditsXmlLoader.java | 7 +-
.../FileDistributionCalculator.java | 18 +-
.../FileDistributionVisitor.java | 16 +-
.../offlineImageViewer/PBImageCorruption.java | 2 +-
.../src/main/proto/InterQJournalProtocol.proto | 7 +
.../src/main/resources/hdfs-default.xml | 27 +-
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 3 +-
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +
.../src/site/markdown/HdfsProvidedStorage.md | 2 +-
.../hadoop/fs/TestEnhancedByteBufferAccess.java | 2 +-
.../hadoop/fs/contract/hdfs/TestDFSWrappedIO.java | 39 +-
.../contract/hdfs/TestHDFSContractBulkDelete.java | 25 +-
.../java/org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +-
.../hadoop/hdfs/TestDecommissionWithStriped.java | 54 +
.../org/apache/hadoop/hdfs/TestFileChecksum.java | 55 +-
.../org/apache/hadoop/hdfs/TestFileCreation.java | 5 +-
.../java/org/apache/hadoop/hdfs/TestHDFSTrash.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestSafeMode.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestSetTimes.java | 5 +-
.../hadoop/hdfs/net/TestDFSNetworkTopology.java | 19 +
.../sasl/TestCustomizedCallbackHandler.java | 138 ++
.../datatransfer/sasl/TestSaslDataTransfer.java | 2 +-
.../server/TestJournalNodeHttpServerXFrame.java | 86 ++
.../hdfs/qjournal/server/TestJournalNodeSync.java | 44 +
.../hadoop/hdfs/server/balancer/TestBalancer.java | 305 +++-
.../balancer/TestBalancerLongRunningTasks.java | 152 ++
.../balancer/TestBalancerWithHANameNodes.java | 4 +-
.../server/blockmanagement/TestBlockManager.java | 8 +-
.../blockmanagement/TestBlockManagerSafeMode.java | 1 +
.../blockmanagement/TestBlockStatsMXBean.java | 26 +
.../TestLowRedundancyBlockQueues.java | 42 +-
.../blockmanagement/TestRedundancyMonitor.java | 2 +-
.../hdfs/server/datanode/TestBPOfferService.java | 13 +-
.../hdfs/server/datanode/TestDirectoryScanner.java | 48 +
.../server/datanode/TestReadOnlySharedStorage.java | 6 +-
.../datanode/fsdataset/impl/TestFsDatasetImpl.java | 44 +-
.../apache/hadoop/hdfs/server/mover/TestMover.java | 148 ++
.../server/namenode/NNThroughputBenchmark.java | 170 ++-
.../hdfs/server/namenode/NameNodeAdapter.java | 100 +-
.../namenode/NameNodeAdapterMockitoUtil.java | 124 ++
.../namenode/TestAuditLoggerWithCommands.java | 20 +
.../hdfs/server/namenode/TestCacheDirectives.java | 3 +-
.../namenode/TestCommitBlockSynchronization.java | 7 +-
.../hdfs/server/namenode/TestFSDirAttrOp.java | 6 +-
.../server/namenode/TestFSDirEncryptionZoneOp.java | 59 +
.../hadoop/hdfs/server/namenode/TestFSImage.java | 17 +-
.../server/namenode/TestFSImageWithSnapshot.java | 40 +-
...TestFSImageWithSnapshotParallelAndCompress.java | 58 +
.../namenode/TestFSNamesystemLockReport.java | 18 +-
.../server/namenode/TestNNThroughputBenchmark.java | 83 +-
.../hdfs/server/namenode/TestNameNodeMXBean.java | 64 +
.../namenode/TestRedundantEditLogInputStream.java | 67 +
.../server/namenode/TestSnapshotPathINodes.java | 4 +-
.../namenode/fgl/FSNLockBenchmarkThroughput.java | 322 -----
.../fgl/TestFSNLockBenchmarkThroughput.java | 104 --
.../server/namenode/ha/TestFailureToReadEdits.java | 3 +-
.../server/namenode/ha/TestHAStateTransitions.java | 3 +-
.../hdfs/server/namenode/ha/TestObserverNode.java | 8 +-
.../server/namenode/ha/TestStandbyCheckpoints.java | 8 +-
.../namenode/metrics/TestNameNodeMetrics.java | 7 +
.../snapshot/TestFileWithSnapshotFeature.java | 19 +-
.../namenode/snapshot/TestSnapshotDiffReport.java | 2 +-
.../web/resources/TestWebHdfsDataLocality.java | 2 +-
.../hdfs/shortcircuit/TestShortCircuitCache.java | 2 +-
.../hdfs/util/TestCombinedHostsFileReader.java | 57 +
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 4 +-
hadoop-mapreduce-project/bin/mapred | 4 +
.../v2/app/rm/preemption/AMPreemptionPolicy.java | 2 +-
.../mapreduce/v2/app/webapp/AMWebServices.java | 15 +-
.../hadoop/mapreduce/v2/app/webapp/ConfBlock.java | 2 +-
.../hadoop/mapreduce/v2/app/TestRecovery.java | 2 +-
.../mapreduce/v2/app/rm/TestRMCommunicator.java | 2 +-
.../v2/app/webapp/TestAMWebServicesAttempts.java | 2 +-
.../v2/app/webapp/TestAMWebServicesJobs.java | 2 +-
.../mapreduce/v2/app/webapp/TestAppController.java | 2 +
.../org/apache/hadoop/mapred/LocalJobRunner.java | 4 +-
.../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 2 +-
.../hadoop/mapred/TestLocalModeWithNewApis.java | 2 +-
.../hadoop/mapred/.LineRecordReader.java.swp | Bin 0 -> 20480 bytes
.../java/org/apache/hadoop/mapred/Counters.java | 2 +-
.../org/apache/hadoop/mapred/FileInputFormat.java | 2 +-
.../org/apache/hadoop/mapred/FileOutputFormat.java | 16 +-
.../hadoop/mapred/InvalidInputException.java | 2 +-
.../java/org/apache/hadoop/mapred/JobConf.java | 4 +-
.../org/apache/hadoop/mapred/LineRecordReader.java | 2 +
.../java/org/apache/hadoop/mapred/MapRunnable.java | 2 +-
.../org/apache/hadoop/mapred/MultiFileSplit.java | 2 +-
.../org/apache/hadoop/mapred/SortedRanges.java | 2 +-
.../java/org/apache/hadoop/mapred/TaskLog.java | 6 +-
.../org/apache/hadoop/mapred/jobcontrol/Job.java | 2 +-
.../hadoop/mapred/join/CompositeInputFormat.java | 12 +-
.../hadoop/mapred/join/CompositeRecordReader.java | 4 +-
.../hadoop/mapred/join/OverrideRecordReader.java | 2 +-
.../java/org/apache/hadoop/mapred/join/Parser.java | 2 +-
.../hadoop/mapred/lib/FieldSelectionMapReduce.java | 2 +-
.../hadoop/mapred/lib/TotalOrderPartitioner.java | 2 +-
.../main/java/org/apache/hadoop/mapreduce/Job.java | 2 +-
.../org/apache/hadoop/mapreduce/JobStatus.java | 2 +-
.../hadoop/mapreduce/TaskCompletionEvent.java | 2 +-
.../mapreduce/jobhistory/JobQueueChangeEvent.java | 12 +-
.../mapreduce/lib/aggregate/ValueHistogram.java | 4 +-
.../lib/fieldsel/FieldSelectionHelper.java | 8 +-
.../lib/input/CombineFileInputFormat.java | 2 +-
.../mapreduce/lib/input/CombineFileSplit.java | 4 +-
.../mapreduce/lib/input/FileInputFormat.java | 2 +-
.../mapreduce/lib/input/InvalidInputException.java | 2 +-
.../mapreduce/lib/input/LineRecordReader.java | 79 +-
.../mapreduce/lib/jobcontrol/ControlledJob.java | 4 +-
.../mapreduce/lib/join/CompositeInputFormat.java | 12 +-
.../mapreduce/lib/join/CompositeRecordReader.java | 4 +-
.../mapreduce/lib/join/OverrideRecordReader.java | 2 +-
.../apache/hadoop/mapreduce/lib/join/Parser.java | 2 +-
.../hadoop/mapreduce/lib/join/TupleWritable.java | 4 +-
.../mapreduce/lib/output/FileOutputFormat.java | 8 +-
.../manifest/ManifestCommitterConfig.java | 24 +
.../manifest/ManifestCommitterConstants.java | 31 +-
.../manifest/ManifestCommitterStatisticNames.java | 9 +
.../committer/manifest/files/ManifestPrinter.java | 4 +-
.../committer/manifest/impl/InternalConstants.java | 9 +
.../manifest/impl/ManifestStoreOperations.java | 29 +
.../ManifestStoreOperationsThroughFileSystem.java | 5 +
.../committer/manifest/stages/AbortTaskStage.java | 7 +-
.../manifest/stages/AbstractJobOrTaskStage.java | 226 ++-
.../committer/manifest/stages/CleanupJobStage.java | 233 ++-
.../committer/manifest/stages/CommitJobStage.java | 8 +-
.../committer/manifest/stages/CommitTaskStage.java | 27 +-
.../stages/CreateOutputDirectoriesStage.java | 11 +-
.../manifest/stages/SaveSuccessFileStage.java | 3 +-
.../manifest/stages/SaveTaskManifestStage.java | 42 +-
.../committer/manifest/stages/SetupJobStage.java | 3 +-
.../committer/manifest/stages/StageConfig.java | 22 +
.../lib/partition/TotalOrderPartitioner.java | 10 +-
.../apache/hadoop/mapreduce/split/JobSplit.java | 2 +-
.../hadoop/mapreduce/task/reduce/Fetcher.java | 8 +-
.../task/reduce/ShuffleSchedulerImpl.java | 2 +-
.../org/apache/hadoop/mapreduce/tools/CLI.java | 2 +-
.../src/site/markdown/MapredCommands.md | 5 +
.../src/site/markdown/manifest_committer.md | 351 ++++-
.../markdown/manifest_committer_architecture.md | 6 +
.../hadoop/mapred/TestFileOutputCommitter.java | 2 +-
.../java/org/apache/hadoop/mapred/TestMapTask.java | 18 +-
.../hadoop/mapred/TestTaskProgressReporter.java | 6 +
.../lib/output/TestFileOutputCommitter.java | 2 +-
.../manifest/AbstractManifestCommitterTest.java | 42 +-
.../manifest/ManifestCommitterTestSupport.java | 16 +
.../committer/manifest/TestCleanupStage.java | 66 +-
.../committer/manifest/TestCommitTaskStage.java | 316 +++-
.../manifest/TestCreateOutputDirectoriesStage.java | 2 +-
.../manifest/TestJobThroughManifestCommitter.java | 5 +-
.../committer/manifest/TestLoadManifestsStage.java | 2 +-
.../impl/UnreliableManifestStoreOperations.java | 136 +-
.../src/test/resources/log4j.properties | 2 +
.../hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java | 2 +-
.../mapreduce/v2/hs/webapp/HsWebServices.java | 11 +-
.../hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java | 2 +-
.../v2/hs/webapp/TestHsWebServicesAcls.java | 46 +-
.../v2/hs/webapp/TestHsWebServicesAttempts.java | 2 +-
.../mapreduce/v2/hs/webapp/VerifyJobsUtils.java | 2 +-
.../org/apache/hadoop/RandomTextWriterJob.java | 2 +-
.../org/apache/hadoop/fs/AccumulatingReducer.java | 10 +-
.../java/org/apache/hadoop/fs/IOMapperBase.java | 4 +-
.../java/org/apache/hadoop/fs/JHLogAnalyzer.java | 44 +-
.../java/org/apache/hadoop/mapred/MRBench.java | 2 +-
.../apache/hadoop/mapred/NotificationTestCase.java | 2 +
.../mapred/TestConcatenatedCompressedInput.java | 2 +-
.../hadoop/mapred/TestFixedLengthInputFormat.java | 2 +-
.../mapred/TestMRCJCFileOutputCommitter.java | 2 +-
.../org/apache/hadoop/mapred/TestMapProgress.java | 2 +-
.../java/org/apache/hadoop/mapred/TestMapRed.java | 2 +-
.../apache/hadoop/mapred/TestMiniMRClasspath.java | 4 +-
.../mapred/TestMultipleTextOutputFormat.java | 8 +-
.../hadoop/mapred/TestOldCombinerGrouping.java | 23 +-
.../apache/hadoop/mapred/TestTextInputFormat.java | 2 +-
.../org/apache/hadoop/mapred/UtilsForTests.java | 6 +-
.../mapred/jobcontrol/JobControlTestUtils.java | 2 +-
.../apache/hadoop/mapreduce/MapReduceTestUtil.java | 6 +-
.../hadoop/mapreduce/MiniHadoopClusterManager.java | 2 +-
.../apache/hadoop/mapreduce/RandomTextWriter.java | 2 +-
.../hadoop/mapreduce/TestNewCombinerGrouping.java | 23 +-
.../mapreduce/TestYarnClientProtocolProvider.java | 5 +-
.../lib/input/TestFixedLengthInputFormat.java | 2 +-
.../lib/output/TestMRCJCFileOutputCommitter.java | 2 +-
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 2 +-
.../hadoop/mapred/uploader/FrameworkUploader.java | 2 +-
.../hadoop-mapreduce-client/pom.xml | 6 +-
.../apache/hadoop/examples/RandomTextWriter.java | 2 +-
.../apache/hadoop/examples/dancing/Pentomino.java | 2 +-
.../org/apache/hadoop/examples/dancing/Sudoku.java | 2 +-
.../org/apache/hadoop/examples/pi/package.html | 71 +-
.../hadoop/examples/terasort/TeraScheduler.java | 4 +-
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-project/pom.xml | 144 +-
.../hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java | 19 +-
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 2 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 2 +-
hadoop-tools/hadoop-archives/pom.xml | 2 +-
.../hadoop-aws/dev-support/findbugs-exclude.xml | 5 -
hadoop-tools/hadoop-aws/pom.xml | 45 +-
.../hadoop/fs/s3a/AWSStatus500Exception.java | 21 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 119 ++
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 53 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 13 +-
.../fs/s3a/ProgressableProgressListener.java | 16 +-
.../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 491 +++++--
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 373 ++---
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 830 ++++++-----
.../org/apache/hadoop/fs/s3a/S3AInputPolicy.java | 24 +-
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 78 +-
.../apache/hadoop/fs/s3a/S3AInstrumentation.java | 11 +
.../org/apache/hadoop/fs/s3a/S3AInternals.java | 12 +-
.../org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 18 +-
.../java/org/apache/hadoop/fs/s3a/S3AStore.java | 305 ++++
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 50 +-
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 78 +-
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 60 +
.../apache/hadoop/fs/s3a/WriteOperationHelper.java | 69 +-
.../org/apache/hadoop/fs/s3a/WriteOperations.java | 13 +-
.../hadoop/fs/s3a/api/PerformanceFlagEnum.java} | 38 +-
.../apache/hadoop/fs/s3a/api/RequestFactory.java | 2 +
.../s3a/audit/impl/AbstractOperationAuditor.java | 4 +-
.../fs/s3a/audit/impl/ActiveAuditManagerS3A.java | 12 +-
.../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 36 +-
.../fs/s3a/auth/AssumedRoleCredentialProvider.java | 5 +
.../delegation/EncryptionSecretOperations.java | 16 +
.../fs/s3a/auth/delegation/EncryptionSecrets.java | 35 +-
.../fs/s3a/commit/AbstractS3ACommitterFactory.java | 5 +-
.../hadoop/fs/s3a/commit/S3ACommitterFactory.java | 7 +-
.../fs/s3a/commit/impl/CommitOperations.java | 98 +-
.../fs/s3a/commit/magic/S3MagicCommitTracker.java | 14 +-
.../apache/hadoop/fs/s3a/impl/AWSClientConfig.java | 43 +-
.../org/apache/hadoop/fs/s3a/impl/AWSHeaders.java | 8 +
.../hadoop/fs/s3a/impl/AwsSdkWorkarounds.java | 59 +
.../fs/s3a/impl/BaseS3AFileSystemOperations.java | 132 ++
.../hadoop/fs/s3a/impl/BulkDeleteOperation.java | 128 ++
.../s3a/impl/BulkDeleteOperationCallbacksImpl.java | 125 ++
.../apache/hadoop/fs/s3a/impl/CSEMaterials.java | 132 ++
.../fs/s3a/impl/CSES3AFileSystemOperations.java | 135 ++
.../org/apache/hadoop/fs/s3a/impl/CSEUtils.java | 198 +++
.../CSEV1CompatibleS3AFileSystemOperations.java | 101 ++
.../apache/hadoop/fs/s3a/impl/ClientManager.java | 83 ++
.../hadoop/fs/s3a/impl/ClientManagerImpl.java | 307 ++++
.../hadoop/fs/s3a/impl/CopyFromLocalOperation.java | 4 +-
.../fs/s3a/impl/EncryptionS3ClientFactory.java | 326 +++++
.../hadoop/fs/s3a/impl/ErrorTranslation.java | 55 +
.../hadoop/fs/s3a/impl/HeaderProcessing.java | 19 +-
.../hadoop/fs/s3a/impl/InternalConstants.java | 7 +
.../fs/s3a/impl/ListingOperationCallbacks.java | 11 +
.../apache/hadoop/fs/s3a/impl/MkdirOperation.java | 77 +-
.../fs/s3a/impl/MultiObjectDeleteException.java | 20 +-
.../hadoop/fs/s3a/impl/ProgressListenerEvent.java | 64 +-
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 56 +
.../apache/hadoop/fs/s3a/impl/S3AEncryption.java | 106 ++
.../fs/s3a/impl/S3AFileSystemOperations.java | 102 ++
.../hadoop/fs/s3a/impl/S3AMultipartUploader.java | 3 +-
.../apache/hadoop/fs/s3a/impl/S3AStoreBuilder.java | 132 ++
.../apache/hadoop/fs/s3a/impl/S3AStoreImpl.java | 811 +++++++++++
.../apache/hadoop/fs/s3a/impl/StoreContext.java | 19 +-
.../hadoop/fs/s3a/impl/StoreContextBuilder.java | 17 +-
...ListenerEvent.java => StoreContextFactory.java} | 18 +-
.../hadoop/fs/s3a/impl/UploadContentProviders.java | 569 ++++++++
.../fs/s3a/impl/logging/Log4JController.java | 52 +
.../hadoop/fs/s3a/impl/logging/LogControl.java | 92 ++
.../fs/s3a/impl/logging/LogControllerFactory.java | 98 ++
.../hadoop/fs/s3a/impl/logging/package-info.java | 15 +-
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 9 +-
.../statistics/BlockOutputStreamStatistics.java | 3 +-
.../s3a/statistics/S3AInputStreamStatistics.java | 4 +
.../statistics/impl/StatisticsFromAwsSdkImpl.java | 43 +
.../markdown/tools/hadoop-aws/assumed_roles.md | 8 +
.../markdown/tools/hadoop-aws/aws_sdk_upgrade.md | 1 +
.../site/markdown/tools/hadoop-aws/committers.md | 2 +-
.../site/markdown/tools/hadoop-aws/connecting.md | 171 ++-
.../site/markdown/tools/hadoop-aws/encryption.md | 316 +++-
.../src/site/markdown/tools/hadoop-aws/index.md | 14 +
.../site/markdown/tools/hadoop-aws/performance.md | 211 ++-
.../src/site/markdown/tools/hadoop-aws/testing.md | 50 +-
.../tools/hadoop-aws/third_party_stores.md | 19 +
.../tools/hadoop-aws/troubleshooting_s3a.md | 369 ++---
.../contract/s3a/ITestS3AContractBulkDelete.java | 230 +++
.../fs/contract/s3a/ITestS3AContractCreate.java | 27 +-
.../fs/contract/s3a/ITestS3AContractMkdir.java | 9 +
.../s3a/ITestS3AContractMkdirWithCreatePerf.java | 68 +
.../fs/contract/s3a/ITestS3AContractRootDir.java | 8 +
.../contract/s3a/ITestS3AContractVectoredRead.java | 4 +-
.../s3a/ITestS3AWrappedIO.java} | 20 +-
.../apache/hadoop/fs/s3a/AbstractS3AMockTest.java | 12 +-
.../hadoop/fs/s3a/AbstractTestS3AEncryption.java | 2 +
.../org/apache/hadoop/fs/s3a/CustomKeyring.java | 72 +
.../apache/hadoop/fs/s3a/EncryptionTestUtils.java | 33 +
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 6 +
.../hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 7 +-
.../fs/s3a/ITestS3ABlockOutputByteBuffer.java | 2 +-
.../hadoop/fs/s3a/ITestS3ABlockOutputDisk.java | 2 +-
.../fs/s3a/ITestS3AClientSideEncryption.java | 165 ++-
.../fs/s3a/ITestS3AClientSideEncryptionCustom.java | 87 ++
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 21 +-
.../hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java | 12 +
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 92 +-
.../fs/s3a/ITestS3AEncryptionSSEKMSDefaultKey.java | 21 +
.../ITestS3AEncryptionSSEKMSUserDefinedKey.java | 4 +-
...stS3AEncryptionSSEKMSWithEncryptionContext.java | 101 ++
.../ITestS3AEncryptionWithDefaultS3Settings.java | 19 +
.../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 71 +-
.../hadoop/fs/s3a/ITestS3AFSMainOperations.java | 6 +-
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 10 +-
.../hadoop/fs/s3a/ITestS3AFileOperationCost.java | 10 +-
.../hadoop/fs/s3a/ITestS3AFileSystemContract.java | 5 +-
.../hadoop/fs/s3a/ITestS3AInputStreamLeakage.java | 161 +++
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 15 +-
.../fs/s3a/ITestS3APrefetchingCacheFiles.java | 8 +-
.../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 25 +-
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3ATestConstants.java | 12 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 66 +-
.../org/apache/hadoop/fs/s3a/TestDataBlocks.java | 233 ++-
.../java/org/apache/hadoop/fs/s3a/TestInvoker.java | 208 ++-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 8 +-
.../hadoop/fs/s3a/TestS3ABlockOutputStream.java | 19 +-
.../apache/hadoop/fs/s3a/TestS3ADeleteOnExit.java | 3 +-
.../hadoop/fs/s3a/TestS3AEndpointParsing.java | 43 +
.../apache/hadoop/fs/s3a/TestSSEConfiguration.java | 69 +-
.../hadoop/fs/s3a/audit/AuditTestSupport.java | 12 +-
.../fs/s3a/audit/TestHttpReferrerAuditHeader.java | 33 +-
.../hadoop/fs/s3a/audit/TestLoggingAuditor.java | 41 +
.../fs/s3a/audit/impl/ReferrerExtractor.java | 52 +
.../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 136 +-
.../fs/s3a/auth/TestMarshalledCredentials.java | 3 +-
.../s3a/auth/delegation/ITestDelegatedMRJob.java | 7 +
.../delegation/ITestSessionDelegationTokens.java | 6 +-
.../delegation/TestS3ADelegationTokenSupport.java | 24 +-
.../fs/s3a/commit/AbstractITCommitProtocol.java | 2 +-
.../fs/s3a/commit/ITestS3ACommitterFactory.java | 234 +--
.../hadoop/fs/s3a/commit/ITestUploadRecovery.java | 276 ++++
.../commit/integration/ITestS3ACommitterMRJob.java | 4 +
.../fs/s3a/commit/staging/StagingTestBase.java | 24 +-
.../s3a/commit/staging/TestStagingCommitter.java | 11 +-
.../fs/s3a/commit/terasort/ITestTerasortOnS3A.java | 22 +-
.../ITestS3AFileContextCreateMkdir.java | 9 +-
... ITestS3AFileContextCreateMkdirCreatePerf.java} | 32 +-
.../ITestS3AFileContextMainOperations.java | 37 +-
.../fileContext/ITestS3AFileContextStatistics.java | 30 +-
.../fs/s3a/fileContext/ITestS3AFileContextURI.java | 6 +-
.../hadoop/fs/s3a/impl/ITestAwsSdkWorkarounds.java | 161 +++
.../fs/s3a/impl/ITestConnectionTimeouts.java | 136 +-
.../hadoop/fs/s3a/impl/TestClientManager.java | 380 +++++
.../hadoop/fs/s3a/impl/TestErrorTranslation.java | 33 +-
.../hadoop/fs/s3a/impl/TestOpenFileSupport.java | 43 +-
.../hadoop/fs/s3a/impl/TestRequestFactory.java | 80 +-
.../hadoop/fs/s3a/impl/TestS3AEncryption.java | 77 +
.../s3a/impl/logging/TestLogControllerFactory.java | 214 +++
.../fs/s3a/performance/AbstractS3ACostTest.java | 5 +-
.../fs/s3a/performance/ITestCreateFileCost.java | 17 +-
.../performance/ITestDirectoryMarkerListing.java | 4 +-
.../fs/s3a/performance/ITestS3ADeleteCost.java | 10 +-
.../fs/s3a/performance/ITestS3AOpenCost.java | 2 +
.../s3a/s3guard/AbstractS3GuardToolTestBase.java | 6 +-
.../hadoop/fs/s3a/s3guard/ITestS3GuardTool.java | 30 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 97 +-
.../fs/s3a/scale/CountingProgressListener.java | 192 +++
.../ITestS3ABlockOutputStreamInterruption.java | 493 +++++++
.../hadoop/fs/s3a/scale/ITestS3AConcurrentOps.java | 99 +-
.../fs/s3a/scale/ITestS3ADirectoryPerformance.java | 13 +-
.../fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java | 10 +-
.../s3a/scale/ITestS3AInputStreamPerformance.java | 9 +-
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 2 +-
.../fs/s3a/statistics/TestErrorCodeMapping.java | 83 ++
.../s3a/test/MinimalListingOperationCallbacks.java | 8 +
.../test/MinimalWriteOperationHelperCallbacks.java | 45 +-
.../hadoop/fs/s3a/test/PublicDatasetTestUtils.java | 12 +
.../hadoop/fs/s3a/test/SdkFaultInjector.java | 262 ++++
.../hadoop/fs/s3a/test/StubS3ClientFactory.java | 122 ++
.../fs/s3a/tools/AbstractMarkerToolTest.java | 3 +-
.../s3a/tools/ITestMarkerToolRootOperations.java | 2 +
.../org/apache/hadoop/fs/s3a/yarn/ITestS3A.java | 41 +-
.../org/apache/hadoop/fs/sdk/TestAWSV2SDK.java | 94 ++
.../hadoop-aws/src/test/resources/log4j.properties | 2 +
hadoop-tools/hadoop-azure/pom.xml | 45 +-
.../src/config/checkstyle-suppressions.xml | 6 +
.../hadoop/fs/azure/NativeAzureFileSystem.java | 9 +-
.../hadoop/fs/azurebfs/AbfsBackoffMetrics.java | 312 ++++
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 276 +++-
.../hadoop/fs/azurebfs/AbfsCountersImpl.java | 102 +-
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 109 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 313 ++--
.../fs/azurebfs/constants/AbfsHttpConstants.java | 92 +-
.../fs/azurebfs/constants/AbfsServiceType.java} | 33 +-
.../fs/azurebfs/constants/AuthConfigurations.java | 4 +
.../fs/azurebfs/constants/ConfigurationKeys.java | 54 +-
.../constants/FileSystemConfigurations.java | 26 +-
.../azurebfs/constants/FileSystemUriSchemes.java | 5 +-
.../constants/HttpHeaderConfigurations.java | 32 +-
.../HttpOperationType.java} | 10 +-
.../fs/azurebfs/constants/HttpQueryParams.java | 26 +
.../AbfsApacheHttpExpect100Exception.java} | 23 +-
...ueException.java => HttpResponseException.java} | 23 +-
.../InvalidConfigurationValueException.java | 4 +
.../services/AppendRequestParameters.java | 59 +
.../contracts/services/AzureServiceErrorCode.java | 20 +-
.../services/BlobAppendRequestParameters.java} | 38 +-
.../fs/azurebfs/oauth2/AzureADAuthenticator.java | 52 +-
.../oauth2/WorkloadIdentityTokenProvider.java | 142 ++
.../fs/azurebfs/services/AbfsAHCHttpOperation.java | 394 +++++
.../fs/azurebfs/services/AbfsApacheHttpClient.java | 144 ++
.../fs/azurebfs/services/AbfsBlobClient.java | 1087 ++++++++++++++
.../hadoop/fs/azurebfs/services/AbfsClient.java | 1520 +++++++++-----------
.../fs/azurebfs/services/AbfsClientContext.java | 2 +-
.../fs/azurebfs/services/AbfsClientHandler.java | 169 +++
.../services/AbfsClientThrottlingIntercept.java | 3 +-
.../azurebfs/services/AbfsConnectionManager.java | 207 +++
.../hadoop/fs/azurebfs/services/AbfsCounters.java | 11 +
.../hadoop/fs/azurebfs/services/AbfsDfsClient.java | 1304 +++++++++++++++++
.../services/AbfsHttpClientConnectionFactory.java | 45 +
.../fs/azurebfs/services/AbfsHttpOperation.java | 536 ++++---
.../fs/azurebfs/services/AbfsInputStream.java | 18 +-
.../fs/azurebfs/services/AbfsJdkHttpOperation.java | 300 ++++
.../services/AbfsManagedApacheHttpConnection.java | 246 ++++
.../services/AbfsManagedHttpClientContext.java | 76 +
.../services/AbfsManagedHttpRequestExecutor.java | 109 ++
.../services/AbfsNoOpThrottlingIntercept.java | 8 +-
.../azurebfs/services/AbfsReadFooterMetrics.java | 549 +++++++
.../fs/azurebfs/services/AbfsRestOperation.java | 280 +++-
.../azurebfs/services/AbfsRestOperationType.java | 15 +
.../azurebfs/services/AbfsThrottlingIntercept.java | 5 +-
.../azurebfs/services/FixedSASTokenProvider.java | 65 +
.../fs/azurebfs/services/KeepAliveCache.java | 306 ++++
.../fs/azurebfs/services/SharedKeyCredentials.java | 17 +-
.../fs/azurebfs/services/TimerFunctionality.java | 4 +-
.../MetricFormat.java} | 23 +-
.../hadoop/fs/azurebfs/utils/NamespaceUtil.java | 88 --
.../hadoop/fs/azurebfs/utils/TracingContext.java | 23 +-
.../apache/hadoop/fs/azurebfs/utils/UriUtils.java | 36 +
.../hadoop-azure/src/site/markdown/abfs.md | 286 +++-
.../hadoop-azure/src/site/markdown/blobEndpoint.md | 102 ++
.../hadoop-azure/src/site/markdown/fns_blob.md | 82 ++
.../hadoop-azure/src/site/markdown/index.md | 2 +
.../src/site/markdown/testing_azure.md | 36 +
.../hadoop-azure/src/site/markdown/wasb.md | 97 ++
.../fs/azurebfs/AbstractAbfsIntegrationTest.java | 37 +-
.../fs/azurebfs/ITestAbfsCustomEncryption.java | 93 +-
.../azurebfs/ITestAbfsInputStreamStatistics.java | 1 -
.../fs/azurebfs/ITestAbfsReadFooterMetrics.java | 404 ++++++
.../fs/azurebfs/ITestAbfsReadWriteAndSeek.java | 64 +-
.../azurebfs/ITestAbfsRestOperationException.java | 82 +-
.../ITestAzureBlobFileSystemAuthorization.java | 43 +-
.../ITestAzureBlobFileSystemCheckAccess.java | 9 +-
.../ITestAzureBlobFileSystemChooseSAS.java | 191 +++
.../azurebfs/ITestAzureBlobFileSystemCreate.java | 9 +-
.../ITestAzureBlobFileSystemDelegationSAS.java | 2 +-
.../azurebfs/ITestAzureBlobFileSystemDelete.java | 4 +-
.../fs/azurebfs/ITestAzureBlobFileSystemE2E.java | 21 +-
.../ITestAzureBlobFileSystemInitAndCreate.java | 88 ++
.../fs/azurebfs/ITestAzureBlobFileSystemLease.java | 25 +-
.../ITestAzureBlobFileSystemListStatus.java | 9 +-
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 132 +-
.../fs/azurebfs/TestAccountConfiguration.java | 122 +-
.../hadoop/fs/azurebfs/TestTracingContext.java | 14 +-
.../fs/azurebfs/commit/AbfsCommitTestHelper.java | 8 +-
.../fs/azurebfs/commit/ITestAbfsTerasort.java | 28 +-
.../contract/ITestAbfsContractBulkDelete.java} | 30 +-
.../contract/ITestAbfsFileSystemContractSeek.java | 2 +
.../fs/azurebfs/contract/ITestAbfsWrappedIO.java | 53 +
.../extensions/MockDelegationSASTokenProvider.java | 11 +-
.../azurebfs/extensions/MockSASTokenProvider.java | 16 +-
.../oauth2/TestWorkloadIdentityTokenProvider.java | 144 ++
.../hadoop/fs/azurebfs/oauth2/package-info.java | 17 +-
.../fs/azurebfs/services/AbfsClientTestUtil.java | 17 +-
.../fs/azurebfs/services/AbfsClientUtils.java | 3 -
.../fs/azurebfs/services/ITestAbfsClient.java | 159 +-
.../azurebfs/services/ITestAbfsClientHandler.java | 49 +
.../ITestAbfsHttpClientRequestExecutor.java | 406 ++++++
.../azurebfs/services/ITestAbfsOutputStream.java | 46 +-
.../azurebfs/services/ITestAbfsRestOperation.java | 138 +-
.../services/ITestApacheClientConnectionPool.java | 131 ++
.../services/ITestExponentialRetryPolicy.java | 32 +-
.../fs/azurebfs/services/TestAbfsClient.java | 141 ++
.../fs/azurebfs/services/TestAbfsInputStream.java | 7 +-
.../fs/azurebfs/services/TestAbfsPerfTracker.java | 36 +-
.../services/TestAbfsRenameRetryRecovery.java | 27 +-
.../azurebfs/services/TestAbfsRestOperation.java | 92 ++
.../TestAbfsRestOperationMockFailures.java | 18 +-
.../services/TestApacheClientConnectionPool.java | 290 ++++
.../services/TestApacheHttpClientFallback.java | 226 +++
.../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 17 +-
.../fs/azurebfs/utils/AccountSASGenerator.java | 103 ++
.../hadoop/fs/azurebfs/utils/SASGenerator.java | 34 +-
.../fs/azurebfs/utils/ServiceSASGenerator.java | 15 +-
.../fs/azurebfs/utils/TracingHeaderValidator.java | 2 +-
.../src/test/resources/log4j.properties | 1 +
.../src/test/resources/workload-identity-pod.yaml | 32 +
hadoop-tools/hadoop-compat-bench/pom.xml | 2 +-
hadoop-tools/hadoop-datajoin/pom.xml | 2 +-
.../apache/hadoop/contrib/utils/join/JobBase.java | 2 +-
hadoop-tools/hadoop-distcp/pom.xml | 4 +-
.../main/java/org/apache/hadoop/tools/DistCp.java | 28 +-
.../java/org/apache/hadoop/tools/DistCpSync.java | 2 +-
.../org/apache/hadoop/tools/mapred/CopyMapper.java | 9 +-
.../mapred/RetriableDirectoryCreateCommand.java | 33 +-
.../tools/mapred/RetriableFileCopyCommand.java | 43 +-
.../org/apache/hadoop/tools/util/DistCpUtils.java | 2 +-
.../hadoop/tools/TestDistCpWithRawXAttrs.java | 130 ++
.../org/apache/hadoop/tools/TestExternalCall.java | 5 +-
.../hadoop-dynamometer-blockgen/pom.xml | 2 +-
.../hadoop-dynamometer-infra/pom.xml | 2 +-
.../hadoop-dynamometer-workload/pom.xml | 2 +-
hadoop-tools/hadoop-extras/pom.xml | 2 +-
hadoop-tools/hadoop-federation-balance/pom.xml | 4 +-
hadoop-tools/hadoop-fs2img/pom.xml | 2 +-
hadoop-tools/hadoop-gridmix/pom.xml | 2 +-
.../org/apache/hadoop/mapred/gridmix/FilePool.java | 2 +-
.../hadoop/mapred/gridmix/TestGridMixClasses.java | 2 +-
hadoop-tools/hadoop-kafka/pom.xml | 2 +-
.../org/apache/hadoop/tools/rumen/JobBuilder.java | 16 +-
.../apache/hadoop/tools/rumen/JobHistoryUtils.java | 6 +-
.../org/apache/hadoop/tools/rumen/LoggedTask.java | 8 +-
.../hadoop/tools/rumen/LoggedTaskAttempt.java | 8 +-
.../hadoop/tools/rumen/datatypes/NodeName.java | 2 +-
hadoop-tools/hadoop-sls/pom.xml | 2 +-
.../java/org/apache/hadoop/yarn/sls/SLSRunner.java | 2 +-
hadoop-tools/hadoop-streaming/pom.xml | 2 +-
.../org/apache/hadoop/streaming/PipeMapRed.java | 2 +-
.../org/apache/hadoop/streaming/StreamJob.java | 2 +-
.../hadoop/streaming/StreamXmlRecordReader.java | 2 -
.../hadoop/streaming/io/IdentifierResolver.java | 2 +-
.../java/org/apache/hadoop/streaming/package.html | 2 +-
.../java/org/apache/hadoop/typedbytes/package.html | 8 +-
.../hadoop/streaming/TestMultipleArchiveFiles.java | 2 +-
.../java/org/apache/hadoop/streaming/UtilTest.java | 2 +-
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 1 +
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 8 +-
.../protocolrecords/SignalContainerRequest.java | 2 +-
.../records/timeline/TimelineEntityGroupId.java | 2 +-
.../yarn/api/resource/PlacementConstraint.java | 10 +-
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 54 +-
.../yarn/conf/TestYarnConfigurationFields.java | 1 -
.../Dockerfile | 2 +-
.../pom.xml | 14 +-
.../application/TestAppCatalogSolrClient.java | 11 +-
.../pom.xml | 2 +-
.../yarn/applications/distributedshell/Client.java | 2 +-
.../unmanagedamlauncher/UnmanagedAMLauncher.java | 2 +-
.../dev-support/findbugs-exclude.xml | 5 +
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop-yarn-services-core/pom.xml | 2 +-
.../hadoop/yarn/service/ServiceScheduler.java | 106 +-
.../timelineservice/ServiceMetricsSink.java | 2 +-
.../containerlaunch/TestAbstractLauncher.java | 2 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 2 +-
.../apache/hadoop/yarn/client/cli/RouterCLI.java | 4 +-
.../apache/hadoop/yarn/client/cli/TestLogsCLI.java | 2 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +-
.../hadoop/yarn/security/AdminACLsManager.java | 4 +-
.../apache/hadoop/yarn/util/BoundedAppender.java | 2 +-
.../hadoop/yarn/util/ProcfsBasedProcessTree.java | 2 +-
.../yarn/util/ResourceCalculatorProcessTree.java | 5 -
.../apache/hadoop/yarn/webapp/view/JQueryUI.java | 4 +-
.../src/main/resources/yarn-default.xml | 66 +
.../logaggregation/TestAggregatedLogFormat.java | 2 +-
.../pom.xml | 6 +-
.../yarn/server/timeline/LeveldbTimelineStore.java | 2 +-
.../timeline/RollingLevelDBTimelineStore.java | 2 +-
.../timeline/security/TimelineACLsManager.java | 2 +-
.../hadoop-yarn-server-common/pom.xml | 16 +-
...uavaCache.java => FederationCaffeineCache.java} | 40 +-
.../federation/cache/FederationGuavaCache.java | 4 +-
.../server/federation/cache/FederationJCache.java | 4 +-
.../LocalityMulticastAMRMProxyPolicy.java | 2 +-
.../policies/dao/WeightedPolicyInfo.java | 2 +-
.../store/impl/ZookeeperFederationStateStore.java | 3 +-
.../store/sql/FederationQueryRunner.java | 2 +-
.../utils/FederationPolicyStoreInputValidator.java | 2 +-
.../federation/utils/FederationRegistryClient.java | 2 +-
.../utils/FederationStateStoreFacade.java | 4 +-
.../hadoop/yarn/server/utils/LeveldbIterator.java | 2 +-
.../federation/cache/TestFederationCache.java | 3 +-
.../impl/TestZookeeperFederationStateStore.java | 3 +-
.../policygenerator/LoadBasedGlobalPolicy.java | 2 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 2 +-
.../yarn/server/nodemanager/ContainerExecutor.java | 3 +-
.../server/nodemanager/DirectoryCollection.java | 158 +-
.../server/nodemanager/LinuxContainerExecutor.java | 25 +-
.../deletion/task/DockerContainerDeletionTask.java | 2 +-
.../privileged/PrivilegedOperationExecutor.java | 2 +-
...java => AbstractCGroupsCpuResourceHandler.java} | 96 +-
...andlerImpl.java => AbstractCGroupsHandler.java} | 280 ++--
...a => AbstractCGroupsMemoryResourceHandler.java} | 89 +-
.../AbstractCGroupsResourceCalculator.java | 212 +++
.../resources/CGroupsBlkioResourceHandlerImpl.java | 2 +-
.../resources/CGroupsCpuResourceHandlerImpl.java | 224 +--
.../linux/resources/CGroupsHandler.java | 93 +-
.../linux/resources/CGroupsHandlerImpl.java | 555 +------
.../CGroupsMemoryResourceHandlerImpl.java | 125 +-
.../linux/resources/CGroupsMountConfig.java | 13 +-
.../linux/resources/CGroupsResourceCalculator.java | 412 ++----
.../resources/CGroupsV2CpuResourceHandlerImpl.java | 105 ++
.../linux/resources/CGroupsV2HandlerImpl.java | 224 +++
.../CGroupsV2MemoryResourceHandlerImpl.java | 51 +
.../resources/CGroupsV2ResourceCalculator.java | 136 ++
.../resources/CombinedResourceCalculator.java | 83 +-
.../resources/NetworkPacketTaggingHandlerImpl.java | 2 +-
.../linux/resources/ResourceHandlerModule.java | 101 +-
.../TrafficControlBandwidthHandlerImpl.java | 4 +-
.../linux/resources/TrafficController.java | 2 +-
.../linux/runtime/docker/DockerCommand.java | 2 +-
.../localizer/LocalCacheDirectoryManager.java | 2 +-
.../localizer/ResourceLocalizationService.java | 14 +-
.../monitor/ContainersMonitorImpl.java | 2 +-
.../com/nvidia/NvidiaGPUPluginForRuntimeV2.java | 2 +-
.../nodemanager/recovery/NMStateStoreService.java | 2 +-
.../util/CgroupsLCEResourcesHandler.java | 4 +-
.../container-executor/impl/container-executor.c | 9 +-
.../nodemanager/TestDirectoryCollection.java | 36 +-
.../TestLinuxContainerExecutorWithMocks.java | 57 +-
.../TestPrivilegedOperationExecutor.java | 2 +-
.../TestCGroupsBlkioResourceHandlerImpl.java | 2 +-
.../TestCGroupsCpuResourceHandlerImpl.java | 15 +-
.../linux/resources/TestCGroupsHandlerBase.java | 147 ++
.../linux/resources/TestCGroupsHandlerImpl.java | 156 +-
.../resources/TestCGroupsResourceCalculator.java | 320 ++---
...va => TestCGroupsV2CpuResourceHandlerImpl.java} | 206 ++-
.../linux/resources/TestCGroupsV2HandlerImpl.java | 420 ++++++
.../resources/TestCGroupsV2ResourceCalculator.java | 110 ++
.../TestCgroupsV2MemoryResourceHandlerImpl.java | 209 +++
.../resources/TestCompareResourceCalculators.java | 2 +-
.../TestNetworkPacketTaggingHandlerImpl.java | 2 +-
.../TestTrafficControlBandwidthHandlerImpl.java | 2 +-
.../linux/resources/TestTrafficController.java | 2 +-
.../resources/gpu/TestGpuResourceAllocator.java | 2 +-
.../linux/runtime/TestDockerContainerRuntime.java | 12 +-
.../runtime/TestHdfsManifestToResourcesPlugin.java | 2 +-
.../runtime/TestImageTagToManifestPlugin.java | 2 +-
.../linux/runtime/TestRuncContainerRuntime.java | 2 +-
.../runtime/docker/TestDockerCommandExecutor.java | 4 +-
.../localizer/TestLocalCacheDirectoryManager.java | 2 +-
.../resourceplugin/com/nec/TestNECVEPlugin.java | 2 +-
.../deviceframework/TestDevicePluginAdapter.java | 2 +-
.../security/TestNMTokenSecretManagerInNM.java | 2 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 7 +-
.../server/resourcemanager/NodesListManager.java | 126 +-
.../server/resourcemanager/ResourceManager.java | 17 +-
.../resourcemanager/ResourceTrackerService.java | 2 +-
.../ProportionalCapacityPreemptionPolicy.java | 2 +-
.../ReservedContainerCandidatesSelector.java | 3 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 2 +-
.../server/resourcemanager/rmnode/RMNodeImpl.java | 15 +-
.../scheduler/AbstractYarnScheduler.java | 200 ++-
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../resourcemanager/scheduler/YarnScheduler.java | 5 +-
.../scheduler/activities/ActivitiesManager.java | 2 +-
.../scheduler/activities/AppAllocation.java | 2 +-
.../scheduler/capacity/AbstractLeafQueue.java | 13 +-
.../scheduler/capacity/AbstractParentQueue.java | 13 +
.../scheduler/capacity/CSQueue.java | 6 +
.../scheduler/capacity/CapacityScheduler.java | 52 +-
.../CapacitySchedulerQueueCapacityHandler.java | 2 +-
.../scheduler/capacity/PlanQueue.java | 2 +-
.../allocator/RegularContainerAllocator.java | 5 +-
.../capacity/preemption/PreemptionManager.java | 6 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 8 +-
.../scheduler/fair/FairScheduler.java | 9 +-
.../scheduler/placement/AppPlacementAllocator.java | 2 +-
.../placement/MultiNodeSortingManager.java | 2 +-
.../resourcemanager/webapp/RMWebServices.java | 2 +-
.../webapp/dao/AllocationTagsInfo.java | 2 +-
.../yarn/server/resourcemanager/RMHATestBase.java | 2 +-
.../resourcemanager/TestRMHAForAsyncScheduler.java | 52 +
.../TestResourceTrackerService.java | 63 +
.../recovery/TestZKRMStateStore.java | 6 +-
.../TestZKRMStateStoreZKClientConnections.java | 2 +-
.../scheduler/TestAbstractYarnScheduler.java | 363 ++++-
.../scheduler/capacity/TestCapacityScheduler.java | 116 +-
.../TestCapacitySchedulerAsyncScheduling.java | 33 +
.../TestCapacitySchedulerAutoQueueCreation.java | 55 +-
.../scheduler/capacity/TestUtils.java | 14 +
.../capacity/conf/TestZKConfigurationStore.java | 3 +-
.../constraint/TestPlacementProcessor.java | 2 +-
.../constraint/algorithm/TestCircularIterator.java | 8 +-
.../scheduler/fair/TestQueuePlacementPolicy.java | 62 +-
.../TestFSConfigToCSConfigArgumentHandler.java | 2 +-
.../converter/TestFSConfigToCSConfigConverter.java | 2 +-
.../converter/TestQueuePlacementConverter.java | 2 +-
.../hadoop-yarn-server-router/pom.xml | 2 +-
.../yarn/server/router/RouterServerUtil.java | 2 +-
.../rmadmin/FederationRMAdminInterceptor.java | 4 +-
.../yarn/server/router/webapp/AppsBlock.java | 2 +-
.../router/webapp/FederationInterceptorREST.java | 10 +-
.../yarn/server/router/webapp/NodesBlock.java | 2 +-
.../yarn/server/router/webapp/RouterBlock.java | 2 +-
.../router/clientrm/TestRouterYarnClientUtils.java | 2 +-
.../clientrm/TestSequentialRouterPolicy.java | 2 +-
.../TestableFederationRMAdminInterceptor.java | 2 +-
.../server/router/secure/TestSecureLogins.java | 2 +-
.../subcluster/TestFederationSubCluster.java | 2 +-
.../hadoop-yarn-server-sharedcachemanager/pom.xml | 2 +-
.../hadoop-yarn-server-tests/pom.xml | 2 +-
.../pom.xml | 2 +-
.../pom.xml | 32 +-
.../TestDocumentStoreCollectionCreator.java | 11 +-
.../TestDocumentStoreTimelineReaderImpl.java | 19 +-
.../TestDocumentStoreTimelineWriterImpl.java | 22 +-
.../cosmosdb/TestCosmosDBDocumentStoreReader.java | 14 +-
.../cosmosdb/TestCosmosDBDocumentStoreWriter.java | 18 +-
.../pom.xml | 4 +-
.../pom.xml | 2 +-
.../pom.xml | 4 +
.../timelineservice/storage/common/BaseTable.java | 2 +-
.../hadoop-yarn-server-timelineservice/pom.xml | 2 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 2 +-
.../src/site/markdown/DockerContainers.md | 2 +-
.../src/site/markdown/Federation.md | 16 +-
.../src/site/markdown/NodeManagerCgroups.md | 56 +-
.../src/site/markdown/PluggableDeviceFramework.md | 2 +-
.../src/site/markdown/ResourceManagerHA.md | 4 +-
.../src/site/markdown/ResourceManagerRestart.md | 4 +-
.../src/site/markdown/RuncContainers.md | 2 +-
.../src/site/markdown/UsingFPGA.md | 2 +-
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 4 +-
.../src/main/webapp/app/models/yarn-app-attempt.js | 2 +-
.../src/main/webapp/app/models/yarn-container.js | 2 +-
.../webapp/app/models/yarn-timeline-appattempt.js | 2 +-
.../webapp/app/models/yarn-timeline-container.js | 2 +-
pom.xml | 19 +-
start-build-env.sh | 2 +-
1075 files changed, 45099 insertions(+), 9203 deletions(-)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoUtils.java
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimerFunctionality.java
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/package-info.java
(87%)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BulkDelete.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BulkDeleteSource.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BulkDeleteUtils.java
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidConfigurationValueException.java
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ClosedIOException.java
(64%)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/WithErasureCoding.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/DefaultBulkDeleteOperation.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FlagSet.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/LeakReporter.java
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidConfigurationValueException.java
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/FileSystemStatisticNames.java
(66%)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/store/ByteBufferInputStream.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/wrappedio/WrappedIO.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/wrappedio/WrappedStatistics.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/wrappedio/impl/DynamicWrappedIO.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/wrappedio/impl/DynamicWrappedStatistics.java
copy
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListenerEvent.java
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/wrappedio/impl/package-info.java
(73%)
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidConfigurationValueException.java
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/wrappedio/package-info.java
(61%)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/CustomizedCallbackHandler.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslMechanismFactory.java
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/package-info.java
(80%)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfigurationHelper.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/dynamic/BindingUtils.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/dynamic/DynConstructors.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/dynamic/DynMethods.java
copy
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ProgressListenerEvent.java
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/dynamic/package-info.java
(63%)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FunctionalIO.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/LazyAtomicReference.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/LazyAutoCloseableReference.java
copy
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/{CallableRaisingIOE.java
=> RunnableRaisingIOE.java} (67%)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/Tuples.java
create mode 100644
hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md
create mode 100644
hadoop-common-project/hadoop-common/src/site/markdown/filesystem/bulkdelete.md
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
=>
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/constants/ConfigConstants.java
(64%)
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimerFunctionality.java
=>
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/constants/package-info.java
(88%)
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoUtils.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractBulkDeleteTest.java
copy
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
=>
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractBulkDelete.java
(77%)
copy
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
=>
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractBulkDelete.java
(77%)
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/TestFlagSet.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/TestLeakReporter.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestErasureCodingEncodeAndDecode.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/wrappedio/impl/TestWrappedIO.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/wrappedio/impl/TestWrappedStatistics.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfigurationHelper.java
copy
hadoop-common-project/hadoop-common/src/{main/java/org/apache/hadoop/util/functional/CallableRaisingIOE.java
=> test/java/org/apache/hadoop/util/TestPureJavaCrc32C.java} (66%)
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/dynamic/Concatenator.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/dynamic/TestDynConstructors.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/dynamic/TestDynMethods.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestFunctionalIO.java
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/functional/TestLazyReferences.java
create mode 100644 hadoop-dist/src/main/compose/hadoop/.env
create mode 100644 hadoop-dist/src/main/compose/hadoop/config
create mode 100644 hadoop-dist/src/main/compose/hadoop/docker-compose.yaml
create mode 100644 hadoop-dist/src/main/docker/Dockerfile
create mode 100644
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_getjni_test.cc
create mode 100644
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/order/LeaderFollowerResolver.java
create mode 100644
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/order/TestLeaderFollowerResolver.java
create mode 100644
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterHttpServerXFrame.java
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/TimerFunctionality.java
=>
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/package-info.java
(87%)
copy
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/FunctionRaisingIOE.java
=>
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestDFSWrappedIO.java
(53%)
copy
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
=>
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractBulkDelete.java
(62%)
create mode 100644
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/TestCustomizedCallbackHandler.java
create mode 100644
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeHttpServerXFrame.java
create mode 100644
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapterMockitoUtil.java
create mode 100644
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirEncryptionZoneOp.java
create mode 100644
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshotParallelAndCompress.java
create mode 100644
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRedundantEditLogInputStream.java
delete mode 100644
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/fgl/FSNLockBenchmarkThroughput.java
delete mode 100644
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/fgl/TestFSNLockBenchmarkThroughput.java
create mode 100644
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/.LineRecordReader.java.swp
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStore.java
copy
hadoop-tools/{hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidConfigurationValueException.java
=>
hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/PerformanceFlagEnum.java}
(62%)
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AwsSdkWorkarounds.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemOperations.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteOperation.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BulkDeleteOperationCallbacksImpl.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEMaterials.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemOperations.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemOperations.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManager.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManagerImpl.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AEncryption.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemOperations.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreBuilder.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java
copy
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/{ProgressListenerEvent.java
=> StoreContextFactory.java} (71%)
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/UploadContentProviders.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/logging/Log4JController.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/logging/LogControl.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/logging/LogControllerFactory.java
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
=>
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/logging/package-info.java
(77%)
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractBulkDelete.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java
copy
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/{s3a/ITestS3ABlockOutputByteBuffer.java
=> contract/s3a/ITestS3AWrappedIO.java} (65%)
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSWithEncryptionContext.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEndpointParsing.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/impl/ReferrerExtractor.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestUploadRecovery.java
copy
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/{ITestS3AFileContextCreateMkdir.java
=> ITestS3AFileContextCreateMkdirCreatePerf.java} (54%)
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestAwsSdkWorkarounds.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestClientManager.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3AEncryption.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/logging/TestLogControllerFactory.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/CountingProgressListener.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ABlockOutputStreamInterruption.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/statistics/TestErrorCodeMapping.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/SdkFaultInjector.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/StubS3ClientFactory.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/sdk/TestAWSV2SDK.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsBackoffMetrics.java
copy
hadoop-tools/{hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java
=>
hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsServiceType.java}
(57%)
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/{services/TimerFunctionality.java
=> constants/HttpOperationType.java} (86%)
copy
hadoop-tools/{hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputByteBuffer.java
=>
hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsApacheHttpExpect100Exception.java}
(59%)
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/{InvalidConfigurationValueException.java
=> HttpResponseException.java} (59%)
copy
hadoop-tools/{hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockOutputDisk.java
=>
hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/BlobAppendRequestParameters.java}
(55%)
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/WorkloadIdentityTokenProvider.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsAHCHttpOperation.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsApacheHttpClient.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClientHandler.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsConnectionManager.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpClientConnectionFactory.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsJdkHttpOperation.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsManagedApacheHttpConnection.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsManagedHttpClientContext.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsManagedHttpRequestExecutor.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/FixedSASTokenProvider.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/KeepAliveCache.java
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/{services/AbfsNoOpThrottlingIntercept.java
=> utils/MetricFormat.java} (60%)
delete mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/NamespaceUtil.java
create mode 100644 hadoop-tools/hadoop-azure/src/site/markdown/blobEndpoint.md
create mode 100644 hadoop-tools/hadoop-azure/src/site/markdown/fns_blob.md
create mode 100644 hadoop-tools/hadoop-azure/src/site/markdown/wasb.md
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java
copy
hadoop-tools/{hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
=>
hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsContractBulkDelete.java}
(59%)
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsWrappedIO.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/TestWorkloadIdentityTokenProvider.java
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
=>
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/package-info.java
(78%)
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsHttpClientRequestExecutor.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestApacheClientConnectionPool.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestApacheClientConnectionPool.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestApacheHttpClientFallback.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AccountSASGenerator.java
create mode 100644
hadoop-tools/hadoop-azure/src/test/resources/workload-identity-pod.yaml
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/cache/{FederationGuavaCache.java
=> FederationCaffeineCache.java} (79%)
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/{CGroupsCpuResourceHandlerImpl.java
=> AbstractCGroupsCpuResourceHandler.java} (68%)
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/{CGroupsHandlerImpl.java
=> AbstractCGroupsHandler.java} (70%)
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/{CGroupsMemoryResourceHandlerImpl.java
=> AbstractCGroupsMemoryResourceHandler.java} (67%)
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/AbstractCGroupsResourceCalculator.java
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsV2CpuResourceHandlerImpl.java
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsV2HandlerImpl.java
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsV2MemoryResourceHandlerImpl.java
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsV2ResourceCalculator.java
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerBase.java
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/{TestCGroupsCpuResourceHandlerImpl.java
=> TestCGroupsV2CpuResourceHandlerImpl.java} (63%)
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsV2HandlerImpl.java
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsV2ResourceCalculator.java
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCgroupsV2MemoryResourceHandlerImpl.java
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
