This is an automated email from the ASF dual-hosted git repository.
stevel pushed a change to branch HADOOP-19256-s3-conditional-writes
in repository https://gitbox.apache.org/repos/asf/hadoop.git
from dc56fc385a4 HADOOP-19295. S3A: large uploads can timeout over slow
links (#7089)
add e4b070025b3 HADOOP-19291. RawLocalFileSystem to allow overlapping
ranges (#7101)
add 9321e322d2a HADOOP-19310. Add JPMS options required by Java 17+
(#7114) Contributed by Cheng Pan.
add 78a08b3b782 MAPREDUCE-7494. File stream leak when LineRecordReader is
interrupted (#7117)
add c63aafd7d1c YARN-11732. Fix potential NPE when calling
SchedulerNode#reservedContainer for CapacityScheduler (#7065). Contributed by
Tao Yang.
add 6589d9f6aae HDFS-17631. Fix RedundantEditLogInputStream.nextOp() state
error when EditLogInputStream.skipUntil() throw IOException (#7066).
Contributed by liuguanghua.
add f931ede86b6 HADOOP-19298. [JDK17] Add a JDK17 profile. (#7085)
Contributed by Shilun Fan.
add 09b348753f8 HDFS-17634. RBF: Fix web UI missing DN last block report
(#7080)
add d1ce9656459 HDFS-17636. Don't add declspec for Windows (#7096)
add 0b3755347c8 HADOOP-19309: S3A: CopyFromLocalFile operation fails when
the source file does not contain file scheme (#7113)
add eb1e30395b3 HDFS-17644:Add log when a node selection is rejected by
BPP UpgradeDomain (#7109)
add 7a7b346b0ab Revert "HADOOP-19298. [JDK17] Add a JDK17 profile. (#7085)
Contributed by Shilun Fan." (#7132)
add 00cddf5bea9 HDFS-17646. Add Option to limit Balancer overUtilized
nodes num in each iteration. (#7120). Contributed by Zhaobo Huang.
add 4f3abd2f482 HDFS-17654. Fix bugs in TestRouterMountTable (#7137).
Contributed by farmmamba.
add e4789a2fd3e HDFS-17607. Reduce the number of times conf is loaded when
DataNode startUp (#7012). Contributed by lei w.
add 66baf1eb51d HADOOP-18682. Move hadoop docker scripts under the main
source code (#6483). Contributed by Christos Bisias.
add df979e70de9 HADOOP-19297. [JDK17] Upgrade maven.plugin-tools.version
to 3.10.2 (#7125) Contributed by Min Yan.
add 9ae01bdbe83 HADOOP-19143. Upgrade commons-cli to 1.9.0 (#7126)
Contributed by Min Yan.
add 51ebc3c20e8 HADOOP-18583. Fix loading of OpenSSL 3.x symbols (#5256)
add f7651e2f63d HADOOP-19243. Upgrade Mockito version to 4.11.0 (#6968)
add 487727a5d10 HADOOP-18960: [ABFS] Making Contract tests run in
sequential and Other Test Fixes (#7104)
add 96572764921 HDFS-17642. Add target node list, exclude source node
list, and exclude target node list parameters to balancer (#7127)
add 9a743bd17f5 HADOOP-19315. Upgrade Apache Avro to 1.11.4 (#7128)
add 2273278d0b0 HADOOP-18708: S3A: Support S3 Client Side Encryption(CSE)
(#6884)
add 7999db55da4 HADOOP-19330. S3A: Add LeakReporter; use in S3AInputStream
(#7151)
add 317db31a9a5 HADOOP-19306. Support user defined auth Callback in
SaslRpcServer. (#7140)
add 7543f3aadab HADOOP-19317. S3A: fs.s3a.connection.expect.continue
controls 100 CONTINUE behavior (#7134)
add c2f13cb7621 HADOOP-18325: [ABFS] Fix metric related test failures due
to missing config (#6847)
add cd2cffe73f9 HADOOP-19342. SaslRpcServer.AuthMethod print INFO messages
in client side. (#7174)
add 31b8b4a8bd5 HADOOP-19336: S3A: Test failures after CSE support added
(#7164)
add b9060fc00df YARN-11738 Modernize SecretManager config (#7144)
add 2ab33c672ea HDFS-17671 Suppress callstack when adding a datanode to
deadnodes (#7180)
add 919bd18c06f HDFS-17641. Add badly distributed blocks metric (#7123)
add 65a5bf3b20c HADOOP-19226: [ABFS][FNSOverBlob] Implementing Azure Rest
APIs on Blob Endpoint for AbfsBlobClient (#6944)
add 964e0897d4a HADOOP-19346. ViewFileSystem.InnerCache: Replaced
ReentrantReadWriteLock with ConcurrentHashMap/putIfAbsent() (#7187)
add 464d7d97f9f HDFS-17668 Treat null SASL negotiated QOP as auth in
DataTransferSasl… (#7171)
add b55c9c7495a HADOOP-19349: S3A : Improve Client Side Encryption
Documentation (#7191)
add 2c2beb715d0 HADOOP-19358. Update command usage of appendToFile. (#7209)
add 86d8fa6d518 HDFS-17679 Use saslClient#hasInitialResponse() instead of
heuristics in SaslParticipant#createFirstMessage() (#7201)
add 2b2acb028c7 HDFS-17682. Fix incorrect command of fs2img tool. (#7206).
Contributed by fuchaohong.
add 93a4dd02cf2 HDFS-17648. Fix BalancerMetrics duplicate registration
issue. (#7133). Contributed by Zhaobo Huang.
add de8b5061d89 HADOOP-19337. Fix ZKFailoverController NPE issue due to
integer overflow in parseInt when initHM. (#7165). Contributed by ConfX.
add 58896f0e135 HADOOP-19339. OutofBounds Exception due to assumption
about buffer size in BlockCompressorStream. Contributed by ConfX.
add aa5fe6f550c YARN-10058. Handle uncaught exception for async-scheduling
threads to prevent scheduler hangs (#7129). Contributed by Tao Yang.
add efb83ecbc34 HADOOP-19360. Disable releases for apache.snapshots repo
(#7218)
add fc42da70cea HADOOP-19357: [ABFS] Optimizations for Retry Handling and
Client Side Throttling (#7216)
add 626455653f5 YARN-11744. Tackle flaky test
testGetRunningContainersToKill (#7227) Contributed by Cheng Pan.
add 5fe4f134afe HADOOP-19366. Install OpenJDK 17 in default ubuntu build
container (#7226) Contributed by Cheng Pan.
add 7f491908020 YARN-7327: Enable asynchronous scheduling by default for
capacity scheduler (#7138)
add 6cb2e866a6c HADOOP-19367. Fix setting final field value on Java 17
(#7228)
add f32a937511e HADOOP-19362. RPC metrics should be updated correctly when
call is defered. (#7224). Contributed by hfutatzhanghb.
add 3bf43b41441 HADOOP-19370. Fix error links of huaweicloud in site
index. (#7229). Contributed by fuchaohong.
add ecc4e00145f HADOOP-19371. JVM GC Metrics supports ZGC pause time and
count (#7230) Contributed by cxzl25.
add 71b3a6477b5 HADOOP-19335. Bump netty to 4.1.116 due to CVE-2024-47535
(#7158). Contributed by PJ Fanning
add d5b836bc34e HDFS-17655. Cannot run HDFS balancer with
BlockPlacementPolicyWithNodeGroup (#7141). Contributed by YUBI LEE.
add 7e673584a12 HADOOP-19207: [ABFS][FNSOverBlob] Response Handling of
Blob Endpoint APIs and Metadata APIs (#7210)
add 305e3e73a91 HADOOP-19380. Update the year to 2025 (#7262). Contributed
by Ayush Saxena.
add 94d6a77c394 HDFS-17496. DataNode supports more fine-grained dataset
lock based on blockid. (#6764). Contributed by farmmamba.
add fd653fa9885 HADOOP-19376. Add fs.hdfs.impl.disable.cache to
core-default.xml. (#7249)
add fd4aa2a0d41 HDFS-17683. Add metrics for acquiring dataset read/write
lock. (#7211). Contributed by farmmamba.
add f65747d4596 YARN-11751. Remove TestAppLevelTimelineCollector. (#7271)
Contributed by Shilun Fan.
add 815ca41c69b HDFS-17080. fix ec connection leak. (#5807)
add 22a9287e4a6 HDFS-17696. Optimize isBlockReplicatedOk method when
scheduleReconStruction parameter is false (#7243)
add fb1bb6429df HDFS-17695. Fix javadoc for FSDirectory#resolvePath
method. (#7238)
add a4872099908 HDFS-17602. RBF: Fix mount point with SPACE order can not
find the available namespace. (#6991). Contributed by Zhongkun Wu.
add d44ac28e326 HADOOP-19278. S3A: Remove option to delete directory
markers (#7052)
add f0ab1e69723 Revert "HDFS-17496. DataNode supports more fine-grained
dataset lock based on…" (#7279)
add 37bdf945d1a YARN-11743. Cgroup v2 support should fall back to v1 when
there are no v2 controllers (#7222)
add 1bd4835a3d1 HADOOP-19305: Fix ProcessEnvironment ClassCastException in
Shell.java (#7106) Contributed by Butao Zhang.
add d2095fae38e HADOOP-19382. [ABFS][FnsOverBlob] Test Fix for
ITestAzureBlobFileSystemInitAndCreate failure (#7269)
add c24d569a6e6 YARN-11752: Global Scheduler: Improve the container
allocation time (#7277) Contributed by Syed Shameerur Rahman.
add 266dad1617d HDFS-17706. TestBlockTokenWithDFSStriped fails due to
closed streams. (#7275). Contribtued by Felix N.
add c3e322860f7 HADOOP-19229. S3A/ABFS: Vector IO on cloud storage:
increase threshold for range merging (#7281)
No new revisions were added by this update.
Summary of changes:
LICENSE-binary | 80 +-
dev-support/docker/pkg-resolver/packages.json | 6 +-
.../hadoop-client-minicluster/pom.xml | 13 +-
.../hadoop-huaweicloud/pom.xml | 18 +-
hadoop-common-project/hadoop-auth/pom.xml | 1 +
hadoop-common-project/hadoop-common/pom.xml | 2 +-
.../hadoop-common/src/main/bin/hadoop-functions.sh | 9 +
.../hadoop/fs/CommonConfigurationKeysPublic.java | 15 +
.../org/apache/hadoop/fs/MultipartUploader.java | 4 +-
.../org/apache/hadoop/fs/PositionedReadable.java | 7 +-
.../org/apache/hadoop/fs/RawLocalFileSystem.java | 7 +-
.../hadoop/fs/impl/AbstractMultipartUploader.java | 2 +-
.../fs/impl/FileSystemMultipartUploader.java | 2 +-
.../org/apache/hadoop/fs/impl/LeakReporter.java | 143 ++
.../hadoop/fs/statistics/StreamStatisticNames.java | 8 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 50 +-
.../org/apache/hadoop/ha/ZKFailoverController.java | 7 +-
.../src/main/java/org/apache/hadoop/io/Sizes.java | 94 ++
.../hadoop/io/compress/BlockCompressorStream.java | 6 +-
.../org/apache/hadoop/ipc/ProtobufRpcEngine.java | 26 +-
.../org/apache/hadoop/ipc/ProtobufRpcEngine2.java | 24 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 63 +-
.../apache/hadoop/metrics2/source/JvmMetrics.java | 7 +
.../security}/CustomizedCallbackHandler.java | 67 +-
.../org/apache/hadoop/security/SaslConstants.java | 45 -
.../hadoop/security/SaslMechanismFactory.java | 69 +
.../org/apache/hadoop/security/SaslRpcServer.java | 66 +-
.../org/apache/hadoop/security/package-info.java | 12 +-
.../hadoop/security/token/SecretManager.java | 46 +-
.../main/java/org/apache/hadoop/util/Shell.java | 4 +-
.../hadoop/util/functional/RunnableRaisingIOE.java | 50 +-
.../src/org/apache/hadoop/crypto/OpensslCipher.c | 68 +-
.../src/main/resources/core-default.xml | 50 +
.../src/site/markdown/FileSystemShell.md | 8 +-
.../src/site/markdown/HadoopDocker.md | 68 +
.../hadoop-common/src/site/markdown/Metrics.md | 5 +
.../src/site/markdown/SingleCluster.md.vm | 6 +
.../site/markdown/filesystem/fsdatainputstream.md | 9 +-
.../hadoop/conf/TestCommonConfigurationFields.java | 1 +
.../apache/hadoop/constants/ConfigConstants.java | 16 +-
.../org/apache/hadoop/constants/package-info.java | 9 +-
.../test/java/org/apache/hadoop/fs/TestPath.java | 4 +
.../AbstractContractMultipartUploaderTest.java | 36 +-
.../contract/AbstractContractVectoredReadTest.java | 38 +-
.../apache/hadoop/fs/contract/ContractOptions.java | 2 +
.../apache/hadoop/fs/impl/TestLeakReporter.java | 165 +++
.../test/java/org/apache/hadoop/io/TestText.java | 4 +
.../io/serializer/avro/TestAvroSerialization.java | 2 +-
.../hadoop/ipc/TestProtoBufRpcServerHandoff.java | 58 +-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 2 +-
.../java/org/apache/hadoop/ipc/TestServer.java | 2 +
.../security/http/TestCrossOriginFilter.java | 10 +-
.../http/TestRestCsrfPreventionFilter.java | 12 +-
.../java/org/apache/hadoop/test/MockitoUtil.java | 9 +
.../org/apache/hadoop/test/ReflectionUtils.java | 25 +
.../src/test/resources/contract/rawlocal.xml | 5 +
hadoop-common-project/hadoop-kms/pom.xml | 2 +-
hadoop-common-project/hadoop-nfs/pom.xml | 2 +-
hadoop-common-project/hadoop-registry/pom.xml | 2 +-
hadoop-dist/pom.xml | 94 ++
.../src/main/compose/hadoop/.env | 4 +-
hadoop-dist/src/main/compose/hadoop/config | 50 +
.../src/main/compose/hadoop/docker-compose.yaml | 32 +-
.../src/main/docker/Dockerfile | 12 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 2 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 6 +-
.../java/org/apache/hadoop/hdfs/StripeReader.java | 40 +-
.../hadoop/hdfs/client/HdfsClientConfigKeys.java | 3 -
.../hadoop/hdfs/protocol/ECBlockGroupStats.java | 26 +-
.../hadoop/hdfs/protocol/ReplicatedBlockStats.java | 23 +-
.../datatransfer/sasl/DataTransferSaslUtil.java | 9 +-
.../datatransfer/sasl/SaslParticipant.java | 13 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 13 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 +-
.../hadoop-hdfs-native-client/pom.xml | 2 +-
.../src/main/native/libhdfs/CMakeLists.txt | 12 +-
.../src/main/native/libhdfspp/tests/CMakeLists.txt | 13 +-
.../native/libhdfspp/tests/libhdfs_getjni_test.cc | 17 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 2 +-
.../server/federation/metrics/FederationMBean.java | 7 +
.../federation/metrics/NamenodeBeanMetrics.java | 17 +
.../hdfs/server/federation/metrics/RBFMetrics.java | 6 +
.../resolver/MembershipNamenodeResolver.java | 2 +
.../federation/resolver/NamenodeStatusReport.java | 15 +
.../resolver/order/AvailableSpaceResolver.java | 7 +-
.../router/NamenodeHeartbeatService.java | 5 +-
.../federation/store/records/MembershipStats.java | 7 +-
.../records/impl/pb/MembershipStatsPBImpl.java | 14 +-
.../src/main/proto/FederationProtocol.proto | 3 +-
.../server/federation/metrics/TestRBFMetrics.java | 4 +
.../resolver/order/TestAvailableSpaceResolver.java | 8 +
.../server/federation/router/TestRouterAdmin.java | 21 +-
.../federation/router/TestRouterMountTable.java | 3 +-
.../server/federation/router/TestRouterRpc.java | 18 +
.../router/TestRouterRpcMultiDestination.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 2 +-
.../datatransfer/sasl/SaslDataTransferServer.java | 20 +-
.../protocol/datatransfer/sasl/package-info.java | 11 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 96 +-
.../hdfs/server/balancer/BalancerParameters.java | 70 +-
.../hdfs/server/blockmanagement/BlockManager.java | 12 +-
.../BlockPlacementPolicyDefault.java | 7 +-
.../BlockPlacementPolicyWithUpgradeDomain.java | 3 +
.../DatanodeAdminBackoffMonitor.java | 17 +-
.../DatanodeAdminDefaultMonitor.java | 15 +-
.../blockmanagement/DatanodeAdminMonitorBase.java | 18 +
.../blockmanagement/LowRedundancyBlocks.java | 20 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 4 +-
.../hdfs/server/datanode/DataSetLockManager.java | 21 +-
.../hadoop/hdfs/server/datanode/DataStorage.java | 4 +-
.../server/datanode/metrics/DataNodeMetrics.java | 12 +
.../hadoop/hdfs/server/namenode/FSDirectory.java | 2 +-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 22 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 5 +-
.../hdfs/server/namenode/NameNodeMXBean.java | 8 +
.../namenode/RedundantEditLogInputStream.java | 2 +
.../namenode/metrics/ReplicatedBlocksMBean.java | 5 +
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 6 +-
.../src/main/resources/hdfs-default.xml | 9 -
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +
.../src/site/markdown/HdfsProvidedStorage.md | 2 +-
.../org/apache/hadoop/hdfs/TestFileCreation.java | 20 +-
.../hdfs/TestStoragePolicyPermissionSettings.java | 24 +-
.../sasl/TestCustomizedCallbackHandler.java | 39 +-
.../hadoop/hdfs/server/balancer/TestBalancer.java | 320 ++++-
.../balancer/TestBalancerLongRunningTasks.java | 152 ++
.../blockmanagement/TestBlockManagerSafeMode.java | 1 +
.../blockmanagement/TestBlockTokenWithDFS.java | 3 +
.../TestLowRedundancyBlockQueues.java | 42 +-
.../hdfs/server/datanode/TestDataNodeMetrics.java | 20 +
.../hdfs/server/namenode/TestCacheDirectives.java | 3 +-
.../namenode/TestCommitBlockSynchronization.java | 7 +-
.../hdfs/server/namenode/TestFSDirAttrOp.java | 6 +-
.../namenode/TestRedundantEditLogInputStream.java | 67 +
.../server/namenode/TestSnapshotPathINodes.java | 4 +-
.../namenode/metrics/TestNameNodeMetrics.java | 7 +
.../snapshot/TestFileWithSnapshotFeature.java | 19 +-
.../mapreduce/v2/app/rm/TestRMCommunicator.java | 2 +-
.../hadoop/mapred/.LineRecordReader.java.swp | Bin 0 -> 20480 bytes
.../org/apache/hadoop/mapred/LineRecordReader.java | 9 +-
.../mapreduce/jobhistory/JobQueueChangeEvent.java | 12 +-
.../mapreduce/lib/input/LineRecordReader.java | 79 +-
.../v2/hs/webapp/TestHsWebServicesAcls.java | 8 +-
.../mapreduce/TestYarnClientProtocolProvider.java | 5 +-
.../hadoop/mapred/uploader/FrameworkUploader.java | 2 +-
.../hadoop-mapreduce-client/pom.xml | 2 +
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-project/pom.xml | 37 +-
hadoop-project/src/site/site.xml | 2 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 2 +-
hadoop-tools/hadoop-archives/pom.xml | 2 +-
hadoop-tools/hadoop-aws/pom.xml | 70 +-
.../java/org/apache/hadoop/fs/s3a/Constants.java | 83 +-
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 19 +-
.../java/org/apache/hadoop/fs/s3a/Listing.java | 62 +-
.../apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 14 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 343 ++---
.../org/apache/hadoop/fs/s3a/S3AInputStream.java | 78 +-
.../apache/hadoop/fs/s3a/S3AInstrumentation.java | 10 +
.../java/org/apache/hadoop/fs/s3a/S3AStore.java | 38 +
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 13 +-
.../org/apache/hadoop/fs/s3a/S3ClientFactory.java | 76 +
.../java/org/apache/hadoop/fs/s3a/Statistic.java | 4 +
.../apache/hadoop/fs/s3a/WriteOperationHelper.java | 28 +-
.../org/apache/hadoop/fs/s3a/WriteOperations.java | 2 +
.../apache/hadoop/fs/s3a/api/RequestFactory.java | 2 +
.../fs/s3a/auth/CredentialProviderListFactory.java | 4 -
.../fs/s3a/commit/impl/CommitOperations.java | 10 +-
.../fs/s3a/commit/magic/S3MagicCommitTracker.java | 4 +-
.../apache/hadoop/fs/s3a/impl/AWSClientConfig.java | 21 +-
.../org/apache/hadoop/fs/s3a/impl/AWSHeaders.java | 5 +
.../fs/s3a/impl/BaseS3AFileSystemOperations.java | 132 ++
.../apache/hadoop/fs/s3a/impl/CSEMaterials.java | 132 ++
.../fs/s3a/impl/CSES3AFileSystemOperations.java | 135 ++
.../org/apache/hadoop/fs/s3a/impl/CSEUtils.java | 198 +++
.../CSEV1CompatibleS3AFileSystemOperations.java | 101 ++
.../apache/hadoop/fs/s3a/impl/ClientManager.java | 8 +
.../hadoop/fs/s3a/impl/ClientManagerImpl.java | 40 +
.../hadoop/fs/s3a/impl/CopyFromLocalOperation.java | 2 +-
.../hadoop/fs/s3a/impl/DirMarkerTracker.java | 26 -
.../apache/hadoop/fs/s3a/impl/DirectoryPolicy.java | 110 --
.../hadoop/fs/s3a/impl/DirectoryPolicyImpl.java | 212 ---
.../fs/s3a/impl/EncryptionS3ClientFactory.java | 326 +++++
.../hadoop/fs/s3a/impl/ErrorTranslation.java | 55 +
.../hadoop/fs/s3a/impl/HeaderProcessing.java | 13 -
.../fs/s3a/impl/ListingOperationCallbacks.java | 11 +
.../apache/hadoop/fs/s3a/impl/MkdirOperation.java | 15 +-
.../hadoop/fs/s3a/impl/PutObjectOptions.java | 38 +-
.../hadoop/fs/s3a/impl/RequestFactoryImpl.java | 5 +
.../fs/s3a/impl/S3AFileSystemOperations.java | 102 ++
.../hadoop/fs/s3a/impl/S3AMultipartUploader.java | 5 +-
.../apache/hadoop/fs/s3a/impl/S3AStoreImpl.java | 111 ++
.../org/apache/hadoop/fs/s3a/s3guard/S3Guard.java | 6 -
.../apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 67 +-
.../s3a/statistics/S3AInputStreamStatistics.java | 4 +
.../org/apache/hadoop/fs/s3a/tools/MarkerTool.java | 93 +-
.../hadoop/fs/s3native/NativeS3FileSystem.java | 148 --
.../apache/hadoop/fs/s3native/S3xLoginHelper.java | 87 +-
.../markdown/tools/hadoop-aws/assumed_roles.md | 11 +-
.../site/markdown/tools/hadoop-aws/connecting.md | 161 ++-
.../markdown/tools/hadoop-aws/directory_markers.md | 466 ++-----
.../site/markdown/tools/hadoop-aws/encryption.md | 286 +++-
.../src/site/markdown/tools/hadoop-aws/index.md | 8 +-
.../site/markdown/tools/hadoop-aws/performance.md | 9 +-
.../src/site/markdown/tools/hadoop-aws/s3guard.md | 7 +-
.../src/site/markdown/tools/hadoop-aws/s3n.md | 52 -
.../src/site/markdown/tools/hadoop-aws/testing.md | 47 -
.../tools/hadoop-aws/troubleshooting_s3a.md | 189 ---
.../fs/contract/s3a/ITestS3AContractCreate.java | 19 +-
.../contract/s3a/ITestS3AContractVectoredRead.java | 29 +-
.../apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 51 -
.../org/apache/hadoop/fs/s3a/CustomKeyring.java | 72 +
.../hadoop/fs/s3a/ITestDowngradeSyncable.java | 5 -
.../fs/s3a/ITestS3AClientSideEncryption.java | 165 ++-
.../fs/s3a/ITestS3AClientSideEncryptionCustom.java | 87 ++
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 10 +-
.../hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java | 12 +
.../ITestS3AEncryptionSSEKMSUserDefinedKey.java | 4 +-
.../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 4 +-
.../hadoop/fs/s3a/ITestS3AFileOperationCost.java | 31 +-
.../hadoop/fs/s3a/ITestS3AInputStreamLeakage.java | 161 +++
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 2 +-
.../fs/s3a/ITestS3APrefetchingCacheFiles.java | 4 -
.../fs/s3a/ITestS3APrefetchingInputStream.java | 4 -
.../fs/s3a/ITestS3APrefetchingLruEviction.java | 1 -
.../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 8 -
.../apache/hadoop/fs/s3a/MultipartTestUtils.java | 4 +-
.../org/apache/hadoop/fs/s3a/S3ATestConstants.java | 6 -
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 40 +-
.../hadoop/fs/s3a/TestS3ABlockOutputStream.java | 6 +-
.../fs/s3a/audit/ITestAuditAccessChecks.java | 4 -
.../hadoop/fs/s3a/audit/ITestAuditManager.java | 4 -
.../fs/s3a/audit/ITestAuditManagerDisabled.java | 4 -
.../fs/s3a/commit/ITestCommitOperationCost.java | 45 +-
.../hadoop/fs/s3a/commit/ITestUploadRecovery.java | 31 +-
.../commit/integration/ITestS3ACommitterMRJob.java | 4 +
.../fileContext/ITestS3AFileContextStatistics.java | 30 +-
.../hadoop/fs/s3a/impl/ITestAwsSdkWorkarounds.java | 1 +
.../fs/s3a/impl/ITestConnectionTimeouts.java | 2 +
.../fs/s3a/impl/ITestPartialRenamesDeletes.java | 4 -
.../hadoop/fs/s3a/impl/ITestRenameDeleteRace.java | 23 +-
.../apache/hadoop/fs/s3a/impl/ITestXAttrCost.java | 4 -
.../hadoop/fs/s3a/impl/TestClientManager.java | 1 +
.../fs/s3a/impl/TestDirectoryMarkerPolicy.java | 163 ---
.../hadoop/fs/s3a/impl/TestErrorTranslation.java | 33 +-
.../hadoop/fs/s3a/impl/TestRequestFactory.java | 20 +-
.../fs/s3a/performance/AbstractS3ACostTest.java | 91 +-
.../fs/s3a/performance/ITestCreateFileCost.java | 6 +-
.../performance/ITestDirectoryMarkerListing.java | 82 +-
.../fs/s3a/performance/ITestS3ADeleteCost.java | 105 +-
.../s3a/performance/ITestS3AMiscOperationCost.java | 7 +-
.../fs/s3a/performance/ITestS3AMkdirCost.java | 23 -
.../fs/s3a/performance/ITestS3AOpenCost.java | 4 -
.../fs/s3a/performance/ITestS3ARenameCost.java | 52 +-
.../fs/s3a/performance/ITestUnbufferDraining.java | 7 -
.../s3a/s3guard/AbstractS3GuardToolTestBase.java | 6 +-
.../fs/s3a/s3guard/TestAuthoritativePath.java | 132 --
.../fs/s3a/scale/ITestS3ADirectoryPerformance.java | 9 +-
.../fs/s3a/scale/ITestS3AHugeFilesNoMultipart.java | 6 +
.../s3a/test/MinimalListingOperationCallbacks.java | 8 +
.../test/MinimalWriteOperationHelperCallbacks.java | 7 -
.../fs/s3a/tools/AbstractMarkerToolTest.java | 95 +-
.../hadoop/fs/s3a/tools/ITestMarkerTool.java | 172 +--
.../hadoop/fs/s3native/TestS3xLoginHelper.java | 205 ---
hadoop-tools/hadoop-azure/pom.xml | 29 +-
.../src/config/checkstyle-suppressions.xml | 4 +
.../hadoop/fs/azurebfs/AbfsConfiguration.java | 47 +-
.../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 54 +-
.../fs/azurebfs/AzureBlobFileSystemStore.java | 224 +--
.../fs/azurebfs/constants/AbfsHttpConstants.java | 80 +-
.../constants/FileSystemConfigurations.java | 8 +-
.../azurebfs/constants/FileSystemUriSchemes.java | 4 +-
.../constants/HttpHeaderConfigurations.java | 31 +-
.../fs/azurebfs/constants/HttpQueryParams.java | 31 +
.../services/AppendRequestParameters.java | 59 +
.../contracts/services/AzureServiceErrorCode.java | 2 +
.../services/BlobAppendRequestParameters.java} | 47 +-
.../services/BlobListResultEntrySchema.java | 238 ++++
.../contracts/services/BlobListResultSchema.java | 71 +
.../contracts/services/BlobListXmlParser.java | 304 ++++
...rySchema.java => DfsListResultEntrySchema.java} | 48 +-
...tResultSchema.java => DfsListResultSchema.java} | 21 +-
.../contracts/services/ListResultEntrySchema.java | 218 +--
.../contracts/services/ListResultSchema.java | 29 +-
.../services/StorageErrorResponseSchema.java | 62 +
.../fs/azurebfs/services/AbfsAHCHttpOperation.java | 32 +-
.../fs/azurebfs/services/AbfsBlobClient.java | 1472 ++++++++++++++++++++
.../hadoop/fs/azurebfs/services/AbfsClient.java | 99 +-
.../fs/azurebfs/services/AbfsClientHandler.java | 62 +-
.../services/AbfsClientThrottlingIntercept.java | 2 +
.../hadoop/fs/azurebfs/services/AbfsDfsClient.java | 195 ++-
.../fs/azurebfs/services/AbfsHttpOperation.java | 283 +++-
.../fs/azurebfs/services/AbfsInputStream.java | 14 +
.../fs/azurebfs/services/AbfsJdkHttpOperation.java | 12 +-
.../fs/azurebfs/services/AbfsRestOperation.java | 35 +-
.../azurebfs/services/AbfsRestOperationType.java | 15 +
.../fs/azurebfs/services/ListingSupport.java | 2 +-
.../apache/hadoop/fs/azurebfs/utils/UriUtils.java | 29 +-
.../hadoop-azure/src/site/markdown/blobEndpoint.md | 102 ++
.../hadoop-azure/src/site/markdown/fns_blob.md | 8 +-
.../apache/hadoop/fs/azurebfs/ITestAbfsClient.java | 6 +-
.../fs/azurebfs/ITestAbfsCustomEncryption.java | 4 +-
.../fs/azurebfs/ITestAbfsReadFooterMetrics.java | 19 +
.../azurebfs/ITestAbfsRestOperationException.java | 53 +-
.../ITestAzureBlobFileSystemAuthorization.java | 40 +-
.../ITestAzureBlobFileSystemChooseSAS.java | 11 +-
.../azurebfs/ITestAzureBlobFileSystemCreate.java | 22 +-
.../azurebfs/ITestAzureBlobFileSystemDelete.java | 10 +-
.../ITestAzureBlobFileSystemInitAndCreate.java | 13 +-
.../ITestAzureBlobFileSystemListStatus.java | 10 +-
.../fs/azurebfs/ITestGetNameSpaceEnabled.java | 4 +
.../contract/ITestAbfsFileSystemContractSeek.java | 2 +
.../extensions/MockDelegationSASTokenProvider.java | 2 +-
.../fs/azurebfs/services/ITestAbfsClient.java | 111 +-
.../azurebfs/services/ITestAbfsClientHandler.java | 49 +
.../services/ITestAbfsPaginatedDelete.java | 3 +-
.../services/ITestExponentialRetryPolicy.java | 32 +-
.../fs/azurebfs/services/TestAbfsPerfTracker.java | 18 +-
.../azurebfs/services/TestAbfsRestOperation.java | 12 +-
.../hadoop/fs/azurebfs/utils/AbfsTestUtils.java | 17 +-
.../hadoop/fs/azurebfs/utils/TestMockHelpers.java | 59 -
.../hadoop/fs/azurebfs/utils/TestUriUtils.java | 60 +
hadoop-tools/hadoop-compat-bench/pom.xml | 2 +-
hadoop-tools/hadoop-datajoin/pom.xml | 2 +-
hadoop-tools/hadoop-distcp/pom.xml | 4 +-
.../hadoop-dynamometer-blockgen/pom.xml | 2 +-
.../hadoop-dynamometer-infra/pom.xml | 2 +-
.../hadoop-dynamometer-workload/pom.xml | 2 +-
hadoop-tools/hadoop-extras/pom.xml | 2 +-
hadoop-tools/hadoop-federation-balance/pom.xml | 4 +-
hadoop-tools/hadoop-fs2img/pom.xml | 2 +-
hadoop-tools/hadoop-gridmix/pom.xml | 2 +-
.../hadoop/mapred/gridmix/TestGridMixClasses.java | 2 +-
hadoop-tools/hadoop-kafka/pom.xml | 2 +-
.../org/apache/hadoop/tools/rumen/JobBuilder.java | 16 +-
.../apache/hadoop/tools/rumen/JobHistoryUtils.java | 6 +-
.../org/apache/hadoop/tools/rumen/LoggedTask.java | 8 +-
.../hadoop/tools/rumen/LoggedTaskAttempt.java | 8 +-
hadoop-tools/hadoop-sls/pom.xml | 2 +-
hadoop-tools/hadoop-streaming/pom.xml | 2 +-
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 2 +-
.../pom.xml | 6 +-
.../application/TestAppCatalogSolrClient.java | 11 +-
.../pom.xml | 2 +-
.../yarn/applications/distributedshell/Client.java | 2 +-
.../unmanagedamlauncher/UnmanagedAMLauncher.java | 2 +-
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop-yarn-services-core/pom.xml | 2 +-
.../containerlaunch/TestAbstractLauncher.java | 2 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 2 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +-
.../hadoop/yarn/event/TestAsyncDispatcher.java | 5 +-
.../hadoop-yarn/hadoop-yarn-csi/pom.xml | 4 +-
.../pom.xml | 2 +-
.../hadoop-yarn-server-common/pom.xml | 2 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 2 +-
.../linux/resources/ResourceHandlerModule.java | 22 +-
.../linux/resources/TestCGroupsHandlerImpl.java | 2 +-
.../linux/resources/TestCGroupsV2HandlerImpl.java | 2 +-
.../resources/gpu/TestGpuResourceAllocator.java | 2 +-
.../linux/runtime/TestDockerContainerRuntime.java | 10 +-
.../resourceplugin/com/nec/TestNECVEPlugin.java | 2 +-
.../security/TestNMTokenSecretManagerInNM.java | 2 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 7 +-
.../ReservedContainerCandidatesSelector.java | 3 +
.../scheduler/AbstractYarnScheduler.java | 5 +-
.../scheduler/capacity/CapacityScheduler.java | 16 +-
.../capacity/CapacitySchedulerConfiguration.java | 2 +-
.../allocator/RegularContainerAllocator.java | 5 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 8 +-
.../fair/converter/FSYarnSiteConverter.java | 2 +-
.../resourcemanager/TestRMHAForAsyncScheduler.java | 52 +
.../scheduler/TestAbstractYarnScheduler.java | 8 +-
.../TestCapacitySchedulerAsyncScheduling.java | 39 +-
.../TestFSConfigToCSConfigArgumentHandler.java | 2 +-
.../converter/TestFSConfigToCSConfigConverter.java | 2 +-
.../fair/converter/TestFSYarnSiteConverter.java | 4 +-
.../converter/TestQueuePlacementConverter.java | 2 +-
.../TestWeightToPercentageConverter.java | 10 +-
.../TestWeightToWeightConverter.java | 8 +-
.../webapp/TestRMWebServicesCapacitySched.java | 1 +
.../src/test/resources/capacity-scheduler.xml | 255 ++++
.../hadoop-yarn-server-router/pom.xml | 2 +-
.../hadoop-yarn-server-sharedcachemanager/pom.xml | 2 +-
.../hadoop-yarn-server-tests/pom.xml | 2 +-
.../pom.xml | 2 +-
.../pom.xml | 28 +-
.../TestDocumentStoreCollectionCreator.java | 11 +-
.../TestDocumentStoreTimelineReaderImpl.java | 19 +-
.../TestDocumentStoreTimelineWriterImpl.java | 22 +-
.../cosmosdb/TestCosmosDBDocumentStoreReader.java | 14 +-
.../cosmosdb/TestCosmosDBDocumentStoreWriter.java | 18 +-
.../pom.xml | 4 +-
.../pom.xml | 2 +-
.../hadoop-yarn-server-timelineservice/pom.xml | 2 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 2 +-
.../src/site/markdown/CapacityScheduler.md | 37 +
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 4 +-
pom.xml | 17 +
401 files changed, 10412 insertions(+), 4702 deletions(-)
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/LeakReporter.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Sizes.java
rename
{hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security}/CustomizedCallbackHandler.java
(50%)
delete mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslConstants.java
create mode 100644
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslMechanismFactory.java
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/package-info.java
(79%)
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperationType.java
=>
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/RunnableRaisingIOE.java
(55%)
create mode 100644
hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java
=>
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/constants/ConfigConstants.java
(63%)
copy
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java
=>
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/constants/package-info.java
(87%)
create mode 100644
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/TestLeakReporter.java
copy
hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
=> hadoop-dist/src/main/compose/hadoop/.env (86%)
create mode 100644 hadoop-dist/src/main/compose/hadoop/config
copy
hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
=> hadoop-dist/src/main/compose/hadoop/docker-compose.yaml (54%)
rename
hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
=> hadoop-dist/src/main/docker/Dockerfile (82%)
rename
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java
=>
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/package-info.java
(87%)
create mode 100644
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRedundantEditLogInputStream.java
create mode 100644
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/.LineRecordReader.java.swp
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemOperations.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEMaterials.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemOperations.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemOperations.java
delete mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicy.java
delete mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/DirectoryPolicyImpl.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java
create mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemOperations.java
delete mode 100644
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
delete mode 100644
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3n.md
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java
create mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AInputStreamLeakage.java
delete mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestDirectoryMarkerPolicy.java
delete mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestAuthoritativePath.java
delete mode 100644
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3xLoginHelper.java
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/{services/AbfsRestOperationType.java
=> contracts/services/BlobAppendRequestParameters.java} (53%)
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/BlobListResultEntrySchema.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/BlobListResultSchema.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/BlobListXmlParser.java
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/{ListResultEntrySchema.java
=> DfsListResultEntrySchema.java} (85%)
copy
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/{ListResultSchema.java
=> DfsListResultSchema.java} (71%)
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/StorageErrorResponseSchema.java
create mode 100644
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java
create mode 100644 hadoop-tools/hadoop-azure/src/site/markdown/blobEndpoint.md
create mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java
delete mode 100644
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java
create mode 100644
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/capacity-scheduler.xml
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
