Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f02de2e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f02de2e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f02de2e Branch: refs/heads/HADOOP-13345 Commit: 5f02de2e029fd739c8504740467b838d1d336e40 Parents: 2d06842 6aeda55 Author: Mingliang Liu <[email protected]> Authored: Mon Jun 5 13:10:05 2017 -0700 Committer: Mingliang Liu <[email protected]> Committed: Mon Jun 5 13:30:29 2017 -0700 ---------------------------------------------------------------------- BUILDING.txt | 2 - LICENSE.txt | 47 +- dev-support/docker/Dockerfile | 2 + hadoop-assemblies/pom.xml | 4 +- .../resources/assemblies/hadoop-yarn-dist.xml | 5 +- hadoop-build-tools/pom.xml | 12 +- .../main/resources/checkstyle/checkstyle.xml | 4 +- hadoop-client-modules/hadoop-client-api/pom.xml | 4 +- .../hadoop-client-check-invariants/pom.xml | 4 +- .../hadoop-client-check-test-invariants/pom.xml | 4 +- .../hadoop-client-integration-tests/pom.xml | 4 +- .../hadoop-client-minicluster/pom.xml | 8 +- .../hadoop-client-runtime/pom.xml | 5 +- hadoop-client-modules/hadoop-client/pom.xml | 4 +- hadoop-client-modules/pom.xml | 2 +- .../hadoop-cloud-storage/pom.xml | 4 +- hadoop-cloud-storage-project/pom.xml | 4 +- .../hadoop-annotations/pom.xml | 4 +- .../hadoop-auth-examples/pom.xml | 4 +- hadoop-common-project/hadoop-auth/pom.xml | 19 +- .../authentication/util/KerberosName.java | 5 +- .../authentication/util/TestKerberosName.java | 4 +- hadoop-common-project/hadoop-common/pom.xml | 9 +- .../src/main/bin/hadoop-functions.sh | 24 - .../src/main/conf/log4j.properties | 37 +- .../org/apache/hadoop/conf/ConfigRedactor.java | 5 +- .../org/apache/hadoop/conf/Configuration.java | 37 +- .../hadoop/crypto/CryptoOutputStream.java | 4 +- .../org/apache/hadoop/crypto/OpensslCipher.java | 4 +- .../hadoop/crypto/key/kms/ValueQueue.java | 2 +- .../org/apache/hadoop/fs/BlockLocation.java | 21 +- .../apache/hadoop/fs/CachingGetSpaceUsed.java | 3 + .../fs/CommonConfigurationKeysPublic.java | 17 +- .../java/org/apache/hadoop/fs/CreateFlag.java | 8 +- .../src/main/java/org/apache/hadoop/fs/DU.java | 3 +- .../apache/hadoop/fs/FSDataOutputStream.java | 10 +- .../hadoop/fs/FSDataOutputStreamBuilder.java | 8 +- .../java/org/apache/hadoop/fs/FileSystem.java | 8 +- .../java/org/apache/hadoop/fs/FileUtil.java | 5 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 2 +- .../java/org/apache/hadoop/fs/FilterFs.java | 4 +- .../main/java/org/apache/hadoop/fs/Options.java | 2 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 2 +- .../apache/hadoop/fs/StreamCapabilities.java | 67 + .../hadoop/fs/http/AbstractHttpFileSystem.java | 153 + .../apache/hadoop/fs/http/HttpFileSystem.java | 28 + .../apache/hadoop/fs/http/HttpsFileSystem.java | 28 + .../org/apache/hadoop/fs/http/package-info.java | 23 + .../apache/hadoop/fs/sftp/SFTPFileSystem.java | 2 +- .../hadoop/fs/shell/CommandWithDestination.java | 2 +- .../org/apache/hadoop/fs/shell/FsUsage.java | 14 +- .../apache/hadoop/fs/shell/XAttrCommands.java | 7 +- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 2 +- .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 2 +- .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 8 +- .../apache/hadoop/ha/ActiveStandbyElector.java | 4 +- .../org/apache/hadoop/ha/HAServiceProtocol.java | 2 +- .../java/org/apache/hadoop/ha/NodeFencer.java | 3 +- .../org/apache/hadoop/ha/PowerShellFencer.java | 154 + .../org/apache/hadoop/http/HttpServer2.java | 48 +- .../main/java/org/apache/hadoop/io/IOUtils.java | 55 +- .../java/org/apache/hadoop/io/SequenceFile.java | 2 +- .../apache/hadoop/io/compress/BZip2Codec.java | 20 +- .../apache/hadoop/io/compress/CodecPool.java | 10 +- .../io/compress/CompressionInputStream.java | 11 +- .../io/compress/CompressionOutputStream.java | 16 +- .../hadoop/io/compress/CompressorStream.java | 3 +- .../hadoop/io/compress/DecompressorStream.java | 7 +- .../compress/zlib/BuiltInGzipDecompressor.java | 2 +- .../hadoop/io/compress/zlib/ZlibCompressor.java | 6 +- .../io/compress/zlib/ZlibDecompressor.java | 2 +- .../hadoop/io/erasurecode/CodecRegistry.java | 182 + .../apache/hadoop/io/erasurecode/CodecUtil.java | 86 +- .../apache/hadoop/io/erasurecode/ECSchema.java | 40 +- .../io/erasurecode/ErasureCodeConstants.java | 1 + .../rawcoder/DummyRawErasureCoderFactory.java | 12 + .../NativeRSRawErasureCoderFactory.java | 13 + .../NativeXORRawErasureCoderFactory.java | 13 + .../rawcoder/RSLegacyRawDecoder.java | 270 + .../rawcoder/RSLegacyRawEncoder.java | 129 + .../RSLegacyRawErasureCoderFactory.java | 51 + .../rawcoder/RSRawDecoderLegacy.java | 270 - .../rawcoder/RSRawEncoderLegacy.java | 129 - .../rawcoder/RSRawErasureCoderFactory.java | 13 + .../RSRawErasureCoderFactoryLegacy.java | 38 - .../rawcoder/RawErasureCoderFactory.java | 12 + .../rawcoder/XORRawErasureCoderFactory.java | 13 + .../hadoop/io/file/tfile/Compression.java | 2 +- .../org/apache/hadoop/io/nativeio/NativeIO.java | 2 +- .../org/apache/hadoop/ipc/CallQueueManager.java | 1 + .../apache/hadoop/ipc/DecayRpcScheduler.java | 33 +- .../apache/hadoop/ipc/DefaultRpcScheduler.java | 4 + .../org/apache/hadoop/ipc/RpcScheduler.java | 2 + .../main/java/org/apache/hadoop/ipc/Server.java | 22 +- .../apache/hadoop/ipc/metrics/RpcMetrics.java | 4 + .../apache/hadoop/metrics2/AbstractMetric.java | 16 +- .../org/apache/hadoop/metrics2/MetricsTag.java | 13 +- .../metrics2/impl/AbstractMetricsRecord.java | 16 +- .../org/apache/hadoop/metrics2/impl/MsInfo.java | 9 +- .../hadoop/metrics2/lib/MetricsInfoImpl.java | 11 +- .../hadoop/metrics2/lib/MetricsRegistry.java | 20 +- .../apache/hadoop/metrics2/package-info.java | 2 +- .../hadoop/metrics2/source/JvmMetricsInfo.java | 9 +- .../hadoop/metrics2/util/MetricsCache.java | 18 +- .../apache/hadoop/metrics2/util/Servers.java | 4 +- .../org/apache/hadoop/net/NetworkTopology.java | 12 +- .../JniBasedUnixGroupsNetgroupMapping.java | 2 +- .../apache/hadoop/security/SaslRpcServer.java | 4 +- .../ShellBasedUnixGroupsNetgroupMapping.java | 2 +- .../hadoop/security/UserGroupInformation.java | 2 +- .../apache/hadoop/security/ssl/SSLFactory.java | 15 +- .../security/ssl/SSLHostnameVerifier.java | 11 - .../web/DelegationTokenAuthenticator.java | 2 +- .../hadoop/service/ServiceStateException.java | 66 +- .../launcher/AbstractLaunchableService.java | 78 + .../HadoopUncaughtExceptionHandler.java | 129 + .../service/launcher/InterruptEscalator.java | 216 + .../hadoop/service/launcher/IrqHandler.java | 178 + .../service/launcher/LaunchableService.java | 95 + .../service/launcher/LauncherArguments.java | 59 + .../service/launcher/LauncherExitCodes.java | 183 + .../launcher/ServiceLaunchException.java | 81 + .../service/launcher/ServiceLauncher.java | 1044 ++++ .../service/launcher/ServiceShutdownHook.java | 112 + .../hadoop/service/launcher/package-info.java | 462 ++ .../apache/hadoop/util/ComparableVersion.java | 29 +- .../org/apache/hadoop/util/DataChecksum.java | 14 +- .../apache/hadoop/util/ExitCodeProvider.java | 35 + .../java/org/apache/hadoop/util/ExitUtil.java | 257 +- .../hadoop/util/GenericOptionsParser.java | 125 +- .../org/apache/hadoop/util/HostsFileReader.java | 263 +- .../org/apache/hadoop/util/StringInterner.java | 37 +- .../org/apache/hadoop/util/StringUtils.java | 54 +- .../org/apache/hadoop/util/SysInfoWindows.java | 2 +- .../org/apache/hadoop/io/nativeio/NativeIO.c | 2 +- .../services/org.apache.hadoop.fs.FileSystem | 2 + ....erasurecode.rawcoder.RawErasureCoderFactory | 18 + .../src/main/resources/core-default.xml | 35 +- .../src/site/markdown/DeprecatedProperties.md | 1 + .../src/site/markdown/FileSystemShell.md | 6 +- .../hadoop-common/src/site/markdown/Metrics.md | 24 +- .../src/site/markdown/RackAwareness.md | 12 +- .../3.0.0-alpha3/CHANGES.3.0.0-alpha3.md | 71 + .../3.0.0-alpha3/RELEASENOTES.3.0.0-alpha3.md | 22 + .../apache/hadoop/conf/TestConfigRedactor.java | 22 +- .../apache/hadoop/conf/TestConfiguration.java | 23 +- .../crypto/TestCryptoStreamsForLocalFS.java | 10 +- .../hadoop/crypto/TestCryptoStreamsNormal.java | 16 +- .../kms/TestLoadBalancingKMSClientProvider.java | 2 +- .../apache/hadoop/fs/FileContextTestHelper.java | 2 +- .../hadoop/fs/FileSystemContractBaseTest.java | 142 +- .../apache/hadoop/fs/FileSystemTestHelper.java | 2 +- .../java/org/apache/hadoop/fs/TestFsShell.java | 3 +- .../org/apache/hadoop/fs/TestGlobPattern.java | 2 +- .../apache/hadoop/fs/TestLocalFileSystem.java | 37 +- .../apache/hadoop/fs/TestSymlinkLocalFS.java | 6 +- .../hadoop/fs/TestSymlinkLocalFSFileSystem.java | 8 +- .../fs/contract/AbstractContractMkdirTest.java | 31 +- .../contract/rawlocal/RawlocalFSContract.java | 2 +- .../hadoop/fs/http/TestHttpFileSystem.java | 67 + .../hadoop/fs/sftp/TestSFTPFileSystem.java | 14 + .../fs/viewfs/ViewFileSystemBaseTest.java | 6 +- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 2 +- .../apache/hadoop/http/TestSSLHttpServer.java | 21 +- .../java/org/apache/hadoop/io/TestIOUtils.java | 2 +- .../apache/hadoop/io/TestSortedMapWritable.java | 2 +- .../apache/hadoop/io/compress/TestCodec.java | 103 +- .../hadoop/io/compress/TestCodecPool.java | 12 +- .../erasurecode/TestCodecRawCoderMapping.java | 20 +- .../io/erasurecode/TestCodecRegistry.java | 173 + .../hadoop/io/erasurecode/TestECSchema.java | 47 +- .../coder/TestHHXORErasureCoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 2 +- .../rawcoder/RawErasureCoderBenchmark.java | 2 +- .../erasurecode/rawcoder/TestDummyRawCoder.java | 4 +- .../rawcoder/TestNativeRSRawCoder.java | 4 +- .../rawcoder/TestNativeXORRawCoder.java | 4 +- .../rawcoder/TestRSLegacyRawCoder.java | 33 + .../io/erasurecode/rawcoder/TestRSRawCoder.java | 4 +- .../rawcoder/TestRSRawCoderInteroperable1.java | 4 +- .../rawcoder/TestRSRawCoderInteroperable2.java | 4 +- .../rawcoder/TestRSRawCoderLegacy.java | 33 - .../erasurecode/rawcoder/TestRawCoderBase.java | 16 +- .../erasurecode/rawcoder/TestXORRawCoder.java | 4 +- .../rawcoder/TestXORRawCoderInteroperable1.java | 4 +- .../rawcoder/TestXORRawCoderInteroperable2.java | 4 +- .../apache/hadoop/io/nativeio/TestNativeIO.java | 6 +- .../io/retry/UnreliableImplementation.java | 2 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 10 +- .../org/apache/hadoop/net/TestNetUtils.java | 10 +- .../security/TestUserGroupInformation.java | 2 +- .../hadoop/security/ssl/TestSSLFactory.java | 10 +- .../apache/hadoop/service/BreakableService.java | 23 +- .../hadoop/service/TestCompositeService.java | 66 +- .../AbstractServiceLauncherTestBase.java | 317 ++ .../launcher/ExitTrackingServiceLauncher.java | 59 + .../service/launcher/TestServiceConf.java | 146 + .../launcher/TestServiceInterruptHandling.java | 118 + .../service/launcher/TestServiceLauncher.java | 213 + .../TestServiceLauncherCreationFailures.java | 83 + .../TestServiceLauncherInnerMethods.java | 95 + .../ExceptionInExecuteLaunchableService.java | 96 + .../testservices/FailInConstructorService.java | 33 + .../testservices/FailInInitService.java | 38 + .../testservices/FailInStartService.java | 37 + .../testservices/FailingStopInStartService.java | 47 + .../testservices/FailureTestService.java | 55 + .../InitInConstructorLaunchableService.java | 63 + .../testservices/LaunchableRunningService.java | 111 + .../testservices/NoArgsAllowedService.java | 64 + .../testservices/NullBindLaunchableService.java | 46 + .../launcher/testservices/RunningService.java | 84 + .../StoppingInStartLaunchableService.java | 49 + .../StringConstructorOnlyService.java | 39 + .../apache/hadoop/test/GenericTestUtils.java | 34 + .../org/apache/hadoop/util/TestClassUtil.java | 2 +- .../apache/hadoop/util/TestHostsFileReader.java | 19 +- .../util/bloom/BloomFilterCommonTester.java | 4 +- .../hadoop/util/bloom/TestBloomFilters.java | 4 +- .../src/test/resources/contract/rawlocal.xml | 32 +- .../src/test/resources/testConf.xml | 2 +- .../scripts/hadoop_finalize_catalina_opts.bats | 56 - hadoop-common-project/hadoop-kms/pom.xml | 8 +- .../hadoop/crypto/key/kms/server/KMS.java | 2 +- .../crypto/key/kms/server/KMSWebServer.java | 7 +- hadoop-common-project/hadoop-minikdc/pom.xml | 4 +- hadoop-common-project/hadoop-nfs/pom.xml | 4 +- .../org/apache/hadoop/mount/MountInterface.java | 2 +- .../apache/hadoop/nfs/nfs3/Nfs3Constant.java | 2 +- .../hadoop/nfs/nfs3/request/SetAttr3.java | 2 +- .../org/apache/hadoop/oncrpc/RpcMessage.java | 2 +- .../org/apache/hadoop/portmap/TestPortmap.java | 4 +- hadoop-common-project/pom.xml | 4 +- hadoop-dist/pom.xml | 4 +- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 37 +- .../hadoop/hdfs/DFSClientFaultInjector.java | 4 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 43 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 105 +- .../hadoop/hdfs/DistributedFileSystem.java | 98 +- .../apache/hadoop/hdfs/StripedDataStreamer.java | 10 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 12 + .../hdfs/client/HdfsClientConfigKeys.java | 4 + .../hdfs/protocol/AddECPolicyResponse.java | 66 + .../hdfs/protocol/ClientDatanodeProtocol.java | 5 + .../hadoop/hdfs/protocol/ClientProtocol.java | 37 +- .../hdfs/protocol/DatanodeVolumeInfo.java | 122 + .../hadoop/hdfs/protocol/DirectoryListing.java | 4 +- .../hdfs/protocol/ErasureCodingPolicy.java | 62 +- .../hadoop/hdfs/protocol/ExtendedBlock.java | 16 +- .../hadoop/hdfs/protocol/HdfsConstants.java | 2 +- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 22 +- .../hdfs/protocol/IllegalECPolicyException.java | 34 + .../datatransfer/DataTransferProtocol.java | 19 +- .../hdfs/protocol/datatransfer/Sender.java | 29 +- .../ClientDatanodeProtocolTranslatorPB.java | 29 + .../ClientNamenodeProtocolTranslatorPB.java | 48 + .../hadoop/hdfs/protocolPB/PBHelperClient.java | 65 +- .../token/block/BlockTokenIdentifier.java | 66 +- .../hdfs/server/protocol/SlowDiskReports.java | 5 +- .../apache/hadoop/hdfs/util/ECPolicyLoader.java | 325 ++ .../src/main/proto/ClientDatanodeProtocol.proto | 9 + .../src/main/proto/ClientNamenodeProtocol.proto | 7 + .../src/main/proto/datatransfer.proto | 4 + .../src/main/proto/erasurecoding.proto | 23 + .../src/main/proto/hdfs.proto | 21 + .../hdfs/protocol/TestErasureCodingPolicy.java | 86 + .../hadoop/hdfs/util/TestECPolicyLoader.java | 313 ++ hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 27 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 4 +- .../fs/http/server/HttpFSServerWebServer.java | 7 +- .../org/apache/hadoop/lib/server/Server.java | 2 +- .../hadoop/lib/server/ServerException.java | 2 +- .../apache/hadoop/lib/lang/TestXException.java | 2 +- .../org/apache/hadoop/lib/wsrs/TestParam.java | 2 +- .../hadoop-hdfs-native-client/pom.xml | 4 +- .../src/main/native/libhdfs-tests/expect.h | 18 + .../libhdfs-tests/test_libhdfs_threaded.c | 16 + .../src/main/native/libhdfs/exception.c | 75 +- .../src/main/native/libhdfs/exception.h | 16 +- .../src/main/native/libhdfs/hdfs.c | 90 + .../src/main/native/libhdfs/include/hdfs/hdfs.h | 61 + .../src/main/native/libhdfs/jni_helper.c | 95 +- .../src/main/native/libhdfs/jni_helper.h | 35 + .../libhdfs/os/posix/thread_local_storage.c | 54 +- .../native/libhdfs/os/thread_local_storage.h | 61 +- .../libhdfs/os/windows/thread_local_storage.c | 50 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 2 +- .../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 2 +- .../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 2 +- .../dev-support/findbugsExcludeFile.xml | 5 + .../jdiff/Apache_Hadoop_HDFS_3.0.0-alpha3.xml | 326 ++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +- .../src/main/conf/user_ec_policies.xml.template | 72 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 19 +- .../hadoop/hdfs/net/DFSNetworkTopology.java | 24 +- .../hadoop/hdfs/net/DFSTopologyNodeImpl.java | 137 + .../hadoop/hdfs/protocol/LayoutVersion.java | 2 +- .../hdfs/protocol/datatransfer/Receiver.java | 20 +- ...tDatanodeProtocolServerSideTranslatorPB.java | 28 + ...tNamenodeProtocolServerSideTranslatorPB.java | 54 +- .../hdfs/qjournal/client/IPCLoggerChannel.java | 2 +- .../qjournal/server/GetJournalEditServlet.java | 16 +- .../hadoop/hdfs/qjournal/server/JNStorage.java | 25 +- .../hadoop/hdfs/qjournal/server/Journal.java | 21 +- .../hdfs/qjournal/server/JournalNode.java | 25 +- .../hdfs/qjournal/server/JournalNodeSyncer.java | 154 +- .../block/BlockPoolTokenSecretManager.java | 34 +- .../token/block/BlockTokenSecretManager.java | 93 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 47 +- .../hadoop/hdfs/server/balancer/KeyManager.java | 41 +- .../hdfs/server/balancer/NameNodeConnector.java | 16 +- .../hdfs/server/blockmanagement/BlockInfo.java | 18 +- .../server/blockmanagement/BlockManager.java | 248 +- .../BlockPlacementPolicyDefault.java | 40 +- .../BlockUnderConstructionFeature.java | 9 +- .../blockmanagement/CorruptReplicasMap.java | 2 +- .../blockmanagement/DatanodeDescriptor.java | 68 +- .../server/blockmanagement/DatanodeManager.java | 36 +- .../blockmanagement/DatanodeStorageInfo.java | 9 +- .../blockmanagement/InvalidateBlocks.java | 17 +- .../blockmanagement/LowRedundancyBlocks.java | 6 +- .../hdfs/server/common/HdfsServerConstants.java | 7 +- .../apache/hadoop/hdfs/server/common/Util.java | 12 +- .../hdfs/server/datanode/BPOfferService.java | 3 +- .../hdfs/server/datanode/BPServiceActor.java | 5 +- .../hdfs/server/datanode/BlockReceiver.java | 25 +- .../hadoop/hdfs/server/datanode/DNConf.java | 7 +- .../hadoop/hdfs/server/datanode/DataNode.java | 80 +- .../server/datanode/DataNodeFaultInjector.java | 6 +- .../server/datanode/DataNodeLayoutVersion.java | 2 +- .../hdfs/server/datanode/DataStorage.java | 22 +- .../hdfs/server/datanode/DataXceiver.java | 113 +- .../hdfs/server/datanode/FileIoProvider.java | 23 +- .../hdfs/server/datanode/LocalReplica.java | 13 + .../server/datanode/ProfilingFileIoEvents.java | 19 +- .../erasurecode/ErasureCodingWorker.java | 3 +- .../erasurecode/StripedBlockReader.java | 4 +- .../erasurecode/StripedBlockWriter.java | 11 +- .../erasurecode/StripedReconstructionInfo.java | 16 +- .../datanode/erasurecode/StripedWriter.java | 5 +- .../AvailableSpaceVolumeChoosingPolicy.java | 20 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 12 +- .../RoundRobinVolumeChoosingPolicy.java | 2 +- .../fsdataset/VolumeChoosingPolicy.java | 5 +- .../impl/FsDatasetAsyncDiskService.java | 14 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 137 +- .../datanode/fsdataset/impl/FsVolumeList.java | 19 +- .../datanode/web/SimpleHttpProxyHandler.java | 4 +- .../hdfs/server/namenode/AclTransformation.java | 7 +- .../hdfs/server/namenode/BackupImage.java | 2 +- .../hdfs/server/namenode/CacheManager.java | 2 +- .../hadoop/hdfs/server/namenode/Content.java | 2 +- .../ContentSummaryComputationContext.java | 94 +- .../namenode/ErasureCodingPolicyManager.java | 143 +- .../hadoop/hdfs/server/namenode/FSDirAclOp.java | 13 +- .../hdfs/server/namenode/FSDirAttrOp.java | 14 +- .../hdfs/server/namenode/FSDirConcatOp.java | 4 +- .../server/namenode/FSDirEncryptionZoneOp.java | 10 +- .../server/namenode/FSDirErasureCodingOp.java | 39 +- .../hdfs/server/namenode/FSDirMkdirOp.java | 4 +- .../hdfs/server/namenode/FSDirRenameOp.java | 8 +- .../hdfs/server/namenode/FSDirSnapshotOp.java | 7 +- .../hdfs/server/namenode/FSDirSymlinkOp.java | 4 +- .../hdfs/server/namenode/FSDirTruncateOp.java | 10 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 31 +- .../hdfs/server/namenode/FSDirXAttrOp.java | 11 +- .../hdfs/server/namenode/FSDirectory.java | 43 +- .../hdfs/server/namenode/FSEditLogLoader.java | 8 +- .../hadoop/hdfs/server/namenode/FSImage.java | 52 +- .../server/namenode/FSImageFormatPBINode.java | 3 +- .../hdfs/server/namenode/FSNamesystem.java | 153 +- .../hdfs/server/namenode/FSNamesystemLock.java | 121 +- .../server/namenode/FileJournalManager.java | 18 +- .../hadoop/hdfs/server/namenode/INode.java | 1 - .../hdfs/server/namenode/INodeDirectory.java | 19 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 8 +- .../hdfs/server/namenode/INodeReference.java | 2 - .../hdfs/server/namenode/INodeSymlink.java | 1 - .../namenode/INodeWithAdditionalFields.java | 2 +- .../hdfs/server/namenode/INodesInPath.java | 78 +- .../hadoop/hdfs/server/namenode/JournalSet.java | 33 +- .../hdfs/server/namenode/LeaseManager.java | 118 +- .../namenode/NNStorageRetentionManager.java | 27 +- .../hadoop/hdfs/server/namenode/NameNode.java | 2 +- .../server/namenode/NameNodeLayoutVersion.java | 2 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 18 +- .../server/namenode/ha/StandbyCheckpointer.java | 7 +- .../snapshot/DirectorySnapshottableFeature.java | 24 +- .../snapshot/DirectoryWithSnapshotFeature.java | 19 +- .../hdfs/server/namenode/snapshot/Snapshot.java | 1 - .../namenode/snapshot/SnapshotManager.java | 24 +- .../protocol/ReceivedDeletedBlockInfo.java | 2 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 122 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 122 +- .../OfflineEditsXmlLoader.java | 2 +- .../offlineImageViewer/ImageLoaderCurrent.java | 10 +- .../java/org/apache/hadoop/hdfs/util/Diff.java | 2 +- .../src/main/resources/hdfs-default.xml | 51 +- .../src/main/webapps/hdfs/dfshealth.html | 2 +- .../src/main/webapps/hdfs/dfshealth.js | 4 +- .../src/site/markdown/HDFSCommands.md | 6 +- .../src/site/markdown/HDFSErasureCoding.md | 34 +- .../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 2 +- .../hadoop-hdfs/src/site/markdown/ViewFs.md | 2 +- .../org/apache/hadoop/TestRefreshCallQueue.java | 88 +- .../hadoop/fs/TestSymlinkHdfsFileContext.java | 2 +- .../hadoop/fs/TestSymlinkHdfsFileSystem.java | 4 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 14 +- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 12 + .../hadoop/hdfs/TestBlockStoragePolicy.java | 143 + .../TestClientProtocolForPipelineRecovery.java | 92 + .../apache/hadoop/hdfs/TestDFSOutputStream.java | 25 + .../TestDFSRSDefault10x4StripedInputStream.java | 34 - ...TestDFSRSDefault10x4StripedOutputStream.java | 35 - ...fault10x4StripedOutputStreamWithFailure.java | 35 - .../org/apache/hadoop/hdfs/TestDFSShell.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 2 +- ...DFSStripedInputStreamWithRandomECPolicy.java | 45 + .../hadoop/hdfs/TestDFSStripedOutputStream.java | 37 +- .../TestDFSStripedOutputStreamWithFailure.java | 2 +- ...tputStreamWithFailureWithRandomECPolicy.java | 45 + ...FSStripedOutputStreamWithRandomECPolicy.java | 45 + .../org/apache/hadoop/hdfs/TestDFSUtil.java | 2 +- .../hdfs/TestDFSXORStripedInputStream.java | 32 - .../hdfs/TestDFSXORStripedOutputStream.java | 34 - ...estDFSXORStripedOutputStreamWithFailure.java | 35 - .../hadoop/hdfs/TestDataTransferProtocol.java | 3 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +- .../hadoop/hdfs/TestErasureCodingPolicies.java | 129 + .../apache/hadoop/hdfs/TestFileCorruption.java | 1 + .../apache/hadoop/hdfs/TestFileCreation.java | 2 +- .../org/apache/hadoop/hdfs/TestGetBlocks.java | 84 +- .../hadoop/hdfs/TestHDFSFileSystemContract.java | 12 +- .../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 33 + .../hadoop/hdfs/TestMaintenanceState.java | 222 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 2 +- .../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +- .../hdfs/TestWriteBlockGetsBlockLengthHint.java | 6 +- .../apache/hadoop/hdfs/UpgradeUtilities.java | 4 +- .../net/TestDFSNetworkTopologyPerformance.java | 524 ++ .../hadoop/hdfs/protocolPB/TestPBHelper.java | 37 +- .../hdfs/qjournal/TestJournalNodeSync.java | 1 + .../qjournal/client/DirectExecutorService.java | 154 + .../hdfs/qjournal/client/TestQJMWithFaults.java | 3 +- .../client/TestQuorumJournalManager.java | 3 +- .../security/token/block/TestBlockToken.java | 186 +- .../hdfs/server/balancer/TestBalancer.java | 108 +- .../server/balancer/TestBalancerRPCDelay.java | 32 + .../hdfs/server/balancer/TestKeyManager.java | 87 + .../blockmanagement/BlockManagerTestUtil.java | 21 + .../blockmanagement/TestBlockManager.java | 73 +- .../TestBlockUnderConstructionFeature.java | 8 +- .../blockmanagement/TestSlowDiskTracker.java | 4 +- .../server/datanode/BlockReportTestBase.java | 2 +- .../server/datanode/SimulatedFSDataset.java | 24 +- .../hdfs/server/datanode/TestBlockRecovery.java | 6 +- .../server/datanode/TestBlockReplacement.java | 2 +- .../datanode/TestDataNodeHotSwapVolumes.java | 40 +- .../server/datanode/TestDataNodeMXBean.java | 4 +- .../server/datanode/TestDataNodeMetrics.java | 7 +- .../datanode/TestDataNodeVolumeMetrics.java | 4 +- .../TestDataXceiverLazyPersistHint.java | 4 +- .../hdfs/server/datanode/TestDiskError.java | 5 +- .../server/datanode/TestFsDatasetCache.java | 62 +- .../server/datanode/TestSimulatedFSDataset.java | 8 +- .../extdataset/ExternalDatasetImpl.java | 14 +- .../TestAvailableSpaceVolumeChoosingPolicy.java | 76 +- .../TestRoundRobinVolumeChoosingPolicy.java | 29 +- .../fsdataset/impl/TestFsDatasetImpl.java | 6 +- .../fsdataset/impl/TestFsVolumeList.java | 2 +- .../fsdataset/impl/TestSpaceReservation.java | 67 +- .../fsdataset/impl/TestWriteToReplica.java | 33 +- .../TestDataNodeOutlierDetectionViaMetrics.java | 8 + .../hdfs/server/namenode/NameNodeAdapter.java | 31 +- .../namenode/TestBlockUnderConstruction.java | 45 + .../hdfs/server/namenode/TestCheckpoint.java | 4 +- .../TestCommitBlockSynchronization.java | 2 +- .../namenode/TestDecommissioningStatus.java | 113 +- .../TestDefaultBlockPlacementPolicy.java | 46 + .../server/namenode/TestEnabledECPolicies.java | 11 +- .../server/namenode/TestFSNamesystemLock.java | 16 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 32 + .../hdfs/server/namenode/TestLeaseManager.java | 286 +- .../namenode/TestMetadataVersionOutput.java | 4 +- .../namenode/TestNameNodeStatusMXBean.java | 5 +- .../namenode/TestNamenodeStorageDirectives.java | 330 ++ .../namenode/TestReconstructStripedBlocks.java | 2 + .../hdfs/server/namenode/TestStartup.java | 5 +- .../namenode/ha/TestFailureToReadEdits.java | 2 +- .../namenode/ha/TestStandbyCheckpoints.java | 60 + .../namenode/metrics/TestNameNodeMetrics.java | 2 +- .../snapshot/TestOpenFilesWithSnapshot.java | 298 ++ .../snapshot/TestRenameWithSnapshots.java | 201 - .../namenode/snapshot/TestSnapshotDeletion.java | 75 - .../snapshot/TestSnapshotDiffReport.java | 188 +- .../namenode/snapshot/TestSnapshotManager.java | 11 +- .../shortcircuit/TestShortCircuitCache.java | 8 +- .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 20 + .../hdfs/web/TestWebHdfsFileSystemContract.java | 18 +- .../hadoop/hdfs/web/TestWebHdfsTokens.java | 8 +- .../hadoop/tools/TestHdfsConfigFields.java | 2 - .../java/org/apache/hadoop/tools/TestTools.java | 5 +- .../test/resources/testErasureCodingConf.xml | 97 + .../src/test/resources/testHDFSConf.xml | 2 +- .../src/test/resources/test_ec_policies.xml | 65 + hadoop-hdfs-project/pom.xml | 4 +- hadoop-mapreduce-project/.gitignore | 1 - hadoop-mapreduce-project/conf/mapred-site.xml | 21 + .../conf/mapred-site.xml.template | 21 - .../hadoop-mapreduce-client-app/pom.xml | 4 +- .../hadoop/mapred/LocalContainerLauncher.java | 46 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 16 +- .../v2/app/launcher/ContainerLauncherImpl.java | 2 +- .../v2/app/job/impl/TestShuffleProvider.java | 1 - .../app/webapp/TestAMWebServicesAttempts.java | 1 - .../hadoop-mapreduce-client-common/pom.xml | 4 +- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 8 +- .../mapred/TestMRWithDistributedCache.java | 2 +- .../hadoop-mapreduce-client-core/pom.xml | 4 +- .../org/apache/hadoop/mapred/AuditLogger.java | 154 - .../apache/hadoop/mapred/FileInputFormat.java | 2 +- .../apache/hadoop/mapred/FileOutputFormat.java | 2 +- .../java/org/apache/hadoop/mapred/JVMId.java | 2 +- .../org/apache/hadoop/mapred/JobClient.java | 2 +- .../java/org/apache/hadoop/mapred/JobConf.java | 7 +- .../apache/hadoop/mapred/JobContextImpl.java | 3 +- .../org/apache/hadoop/mapred/JobInProgress.java | 2 +- .../org/apache/hadoop/mapred/JobTracker.java | 2 +- .../org/apache/hadoop/mapred/Operation.java | 14 +- .../java/org/apache/hadoop/mapred/Task.java | 2 +- .../hadoop/mapred/TaskAttemptContextImpl.java | 3 +- .../java/org/apache/hadoop/mapred/TaskLog.java | 2 +- .../org/apache/hadoop/mapred/TaskStatus.java | 4 +- .../hadoop/mapred/pipes/BinaryProtocol.java | 2 +- .../org/apache/hadoop/mapreduce/Cluster.java | 2 +- .../java/org/apache/hadoop/mapreduce/Job.java | 4 +- .../org/apache/hadoop/mapreduce/JobStatus.java | 2 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 6 + .../mapreduce/lib/input/FileInputFormat.java | 2 +- .../mapreduce/lib/jobcontrol/ControlledJob.java | 2 +- .../mapreduce/lib/jobcontrol/JobControl.java | 2 +- .../mapreduce/lib/output/FileOutputFormat.java | 2 +- .../hadoop/mapreduce/task/reduce/Fetcher.java | 2 +- .../hadoop/mapreduce/task/reduce/MapHost.java | 2 +- .../org/apache/hadoop/mapred/TestClock.java | 2 +- .../apache/hadoop/mapred/TestClusterStatus.java | 4 +- .../org/apache/hadoop/mapred/TestJobConf.java | 2 +- .../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +- .../hadoop-mapreduce-client-hs/pom.xml | 4 +- .../hadoop/mapreduce/v2/hs/HSAuditLogger.java | 2 +- .../mapreduce/v2/hs/HistoryFileManager.java | 7 +- .../mapreduce/v2/hs/JobHistoryServer.java | 3 - .../hadoop-mapreduce-client-jobclient/pom.xml | 4 +- .../org/apache/hadoop/mapred/YARNRunner.java | 132 +- .../org/apache/hadoop/RandomTextWriterJob.java | 2 +- .../org/apache/hadoop/fs/JHLogAnalyzer.java | 2 +- .../java/org/apache/hadoop/fs/TestDFSIO.java | 4 +- .../apache/hadoop/fs/slive/OperationOutput.java | 2 +- .../org/apache/hadoop/fs/slive/PathFinder.java | 2 +- .../apache/hadoop/hdfs/NNBenchWithoutMR.java | 6 +- .../hadoop/mapred/GenericMRLoadGenerator.java | 2 +- .../java/org/apache/hadoop/mapred/MRBench.java | 2 +- .../apache/hadoop/mapred/TestAuditLogger.java | 161 - .../mapred/TestMRTimelineEventHandling.java | 2 +- .../apache/hadoop/mapred/TestYARNRunner.java | 167 + .../hadoop/mapred/ThreadedMapBenchmark.java | 2 +- .../mapreduce/GenericMRLoadGenerator.java | 2 +- .../hadoop/mapreduce/GrowingSleepJob.java | 68 + .../apache/hadoop/mapreduce/LargeSorter.java | 2 +- .../hadoop/mapreduce/RandomTextWriter.java | 2 +- .../apache/hadoop/mapreduce/RandomWriter.java | 2 +- .../mapreduce/TimelineServicePerformance.java | 2 +- .../mapreduce/security/TestMRCredentials.java | 3 +- .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 6 +- .../apache/hadoop/test/MapredTestDriver.java | 3 + .../hadoop-mapreduce-client-nativetask/pom.xml | 4 +- .../src/CMakeLists.txt | 2 +- .../hadoop-mapreduce-client-shuffle/pom.xml | 4 +- .../hadoop-mapreduce-client/pom.xml | 4 +- .../hadoop-mapreduce-examples/pom.xml | 4 +- .../hadoop/examples/RandomTextWriter.java | 2 +- .../apache/hadoop/examples/RandomWriter.java | 2 +- .../org/apache/hadoop/examples/WordMean.java | 4 +- .../org/apache/hadoop/examples/WordMedian.java | 4 +- .../hadoop/examples/WordStandardDeviation.java | 4 +- .../org/apache/hadoop/examples/pi/Parser.java | 8 +- .../hadoop/examples/terasort/TeraGen.java | 2 +- hadoop-mapreduce-project/pom.xml | 4 +- hadoop-maven-plugins/pom.xml | 2 +- .../resource/ServicesResourceTransformer.java | 4 +- hadoop-minicluster/pom.xml | 4 +- hadoop-project-dist/pom.xml | 24 +- hadoop-project/pom.xml | 36 +- hadoop-tools/hadoop-aliyun/pom.xml | 10 +- .../oss/TestAliyunOSSFileSystemContract.java | 142 +- hadoop-tools/hadoop-archive-logs/pom.xml | 9 +- hadoop-tools/hadoop-archives/pom.xml | 4 +- hadoop-tools/hadoop-aws/pom.xml | 22 +- .../org/apache/hadoop/fs/s3a/Constants.java | 29 +- .../hadoop/fs/s3a/S3AEncryptionMethods.java | 13 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 93 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 120 +- .../src/site/markdown/tools/hadoop-aws/index.md | 111 +- .../site/markdown/tools/hadoop-aws/testing.md | 4 +- .../fs/s3a/AbstractTestS3AEncryption.java | 4 + .../fs/s3a/ITestS3ACopyFromLocalFile.java | 154 + .../ITestS3AEncryptionAlgorithmValidation.java | 4 +- .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 308 +- .../hadoop/fs/s3a/ITestS3AFailureHandling.java | 51 + .../fs/s3a/ITestS3AFileSystemContract.java | 21 +- .../hadoop/fs/s3a/TestSSEConfiguration.java | 223 + .../NativeS3FileSystemContractBaseTest.java | 26 +- hadoop-tools/hadoop-azure-datalake/pom.xml | 12 +- .../src/site/markdown/index.md | 4 +- .../fs/adl/live/AdlStorageConfiguration.java | 8 +- .../adl/live/TestAdlFileSystemContractLive.java | 25 +- hadoop-tools/hadoop-azure/pom.xml | 12 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 195 +- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 26 +- .../fs/azure/RemoteWasbAuthorizerImpl.java | 45 +- .../fs/azure/SyncableDataOutputStream.java | 12 +- .../fs/azure/WasbAuthorizationOperations.java | 2 - .../fs/azure/WasbAuthorizerInterface.java | 3 +- .../fs/azure/security/WasbTokenRenewer.java | 6 - .../hadoop-azure/src/site/markdown/index.md | 90 +- .../hadoop/fs/azure/AbstractWasbTestBase.java | 7 +- .../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 73 +- .../TestNativeAzureFileSystemAuthorization.java | 576 +- ...veAzureFileSystemAuthorizationWithOwner.java | 122 + ...stNativeAzureFileSystemContractEmulator.java | 20 +- .../TestNativeAzureFileSystemContractLive.java | 32 +- ...TestNativeAzureFileSystemContractMocked.java | 11 +- ...tiveAzureFileSystemContractPageBlobLive.java | 25 +- .../fs/azure/TestWasbRemoteCallHelper.java | 18 +- .../TestAzureFileSystemInstrumentation.java | 3 +- hadoop-tools/hadoop-datajoin/pom.xml | 4 +- hadoop-tools/hadoop-distcp/pom.xml | 4 +- .../apache/hadoop/tools/DistCpConstants.java | 10 +- .../org/apache/hadoop/tools/DistCpContext.java | 4 + .../apache/hadoop/tools/DistCpOptionSwitch.java | 8 + .../org/apache/hadoop/tools/DistCpOptions.java | 23 +- .../org/apache/hadoop/tools/OptionsParser.java | 12 + .../tools/mapred/RetriableFileCopyCommand.java | 11 +- .../src/site/markdown/DistCp.md.vm | 1 + .../apache/hadoop/tools/TestDistCpOptions.java | 19 +- hadoop-tools/hadoop-extras/pom.xml | 4 +- hadoop-tools/hadoop-gridmix/pom.xml | 4 +- .../hadoop/mapred/gridmix/InputStriper.java | 6 +- .../TotalHeapUsageEmulatorPlugin.java | 16 +- .../hadoop/mapred/gridmix/GridmixTestUtils.java | 31 +- .../gridmix/TestGridmixMemoryEmulation.java | 2 +- .../mapred/gridmix/TestGridmixSubmission.java | 3 +- hadoop-tools/hadoop-kafka/pom.xml | 12 +- .../hadoop/metrics2/impl/TestKafkaMetrics.java | 10 +- hadoop-tools/hadoop-openstack/pom.xml | 19 +- .../SwiftAuthenticationFailedException.java | 6 +- .../exceptions/SwiftBadRequestException.java | 6 +- .../SwiftInvalidResponseException.java | 9 +- .../SwiftThrottledRequestException.java | 6 +- .../apache/hadoop/fs/swift/http/CopyMethod.java | 41 - .../hadoop/fs/swift/http/CopyRequest.java | 41 + .../swift/http/HttpInputStreamWithRelease.java | 25 +- .../hadoop/fs/swift/http/SwiftRestClient.java | 805 +-- .../snative/SwiftNativeFileSystemStore.java | 11 +- .../hadoop/fs/swift/util/HttpResponseUtils.java | 121 + .../fs/swift/TestSwiftFileSystemContract.java | 19 +- .../TestSwiftFileSystemPartitionedUploads.java | 2 +- .../fs/swift/http/TestSwiftRestClient.java | 2 +- .../src/test/resources/log4j.properties | 3 - hadoop-tools/hadoop-pipes/pom.xml | 4 +- hadoop-tools/hadoop-rumen/pom.xml | 4 +- .../org/apache/hadoop/tools/rumen/TaskInfo.java | 29 +- .../apache/hadoop/tools/rumen/ZombieJob.java | 9 +- .../util/MapReduceJobPropertiesParser.java | 10 +- hadoop-tools/hadoop-sls/pom.xml | 23 +- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 46 +- .../src/main/data/2jobs2min-rumen-jh.json | 587 +-- .../hadoop/yarn/sls/ReservationClientUtil.java | 78 + .../org/apache/hadoop/yarn/sls/SLSRunner.java | 885 +++- .../hadoop/yarn/sls/appmaster/AMSimulator.java | 131 +- .../yarn/sls/appmaster/MRAMSimulator.java | 9 +- .../sls/resourcemanager/MockAMLauncher.java | 5 + .../sls/scheduler/FairSchedulerMetrics.java | 20 +- .../sls/scheduler/SLSCapacityScheduler.java | 33 +- .../yarn/sls/scheduler/SLSFairScheduler.java | 31 +- .../yarn/sls/scheduler/SchedulerWrapper.java | 4 + .../hadoop/yarn/sls/scheduler/TaskRunner.java | 9 +- .../hadoop/yarn/sls/synthetic/SynthJob.java | 306 ++ .../yarn/sls/synthetic/SynthJobClass.java | 180 + .../sls/synthetic/SynthTraceJobProducer.java | 316 ++ .../hadoop/yarn/sls/synthetic/SynthUtils.java | 101 + .../yarn/sls/synthetic/SynthWorkload.java | 121 + .../hadoop/yarn/sls/synthetic/package-info.java | 22 + .../apache/hadoop/yarn/sls/utils/SLSUtils.java | 52 +- .../apache/hadoop/yarn/sls/web/SLSWebApp.java | 37 +- .../src/site/markdown/SchedulerLoadSimulator.md | 150 +- .../hadoop/yarn/sls/BaseSLSRunnerTest.java | 120 + .../sls/TestReservationSystemInvariants.java | 73 + .../apache/hadoop/yarn/sls/TestSLSRunner.java | 90 +- .../hadoop/yarn/sls/TestSynthJobGeneration.java | 96 + .../yarn/sls/appmaster/TestAMSimulator.java | 2 +- .../yarn/sls/scheduler/TestTaskRunner.java | 2 +- .../hadoop/yarn/sls/utils/TestSLSUtils.java | 30 + .../hadoop/yarn/sls/web/TestSLSWebApp.java | 28 +- .../src/test/resources/capacity-scheduler.xml | 10 + .../src/test/resources/fair-scheduler.xml | 8 +- .../hadoop-sls/src/test/resources/inputsls.json | 55 + .../hadoop-sls/src/test/resources/nodes.json | 84 + .../src/test/resources/sls-runner.xml | 6 +- .../hadoop-sls/src/test/resources/syn.json | 53 + .../hadoop-sls/src/test/resources/yarn-site.xml | 10 +- hadoop-tools/hadoop-streaming/pom.xml | 4 +- hadoop-tools/hadoop-tools-dist/pom.xml | 4 +- hadoop-tools/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +- .../yarn/api/ContainerManagementProtocol.java | 8 +- .../yarn/api/records/QueueConfigurations.java | 150 + .../hadoop/yarn/api/records/QueueInfo.java | 42 + .../records/timelineservice/TimelineMetric.java | 2 +- .../hadoop/yarn/conf/YarnConfiguration.java | 8 + .../yarn/exceptions/ConfigurationException.java | 45 + .../src/main/proto/yarn_protos.proto | 14 + .../pom.xml | 4 +- .../distributedshell/ApplicationMaster.java | 4 +- .../pom.xml | 4 +- .../hadoop-yarn-applications/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +- .../apache/hadoop/yarn/client/api/NMClient.java | 86 +- .../yarn/client/api/async/NMClientAsync.java | 98 +- .../api/async/impl/NMClientAsyncImpl.java | 264 +- .../yarn/client/api/impl/NMClientImpl.java | 91 + .../yarn/client/api/impl/YarnClientImpl.java | 10 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 58 +- .../hadoop/yarn/client/ProtocolHATestBase.java | 6 +- .../hadoop/yarn/client/TestRMFailover.java | 5 +- .../api/async/impl/TestNMClientAsync.java | 359 +- .../yarn/client/api/impl/TestAMRMClient.java | 2 +- .../yarn/client/api/impl/TestNMClient.java | 141 + .../hadoop/yarn/client/cli/TestLogsCLI.java | 2 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 4 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 +- .../pb/ApplicationSubmissionContextPBImpl.java | 3 +- .../impl/pb/QueueConfigurationsPBImpl.java | 137 + .../api/records/impl/pb/QueueInfoPBImpl.java | 98 +- .../hadoop/yarn/client/ClientRMProxy.java | 4 +- .../ConfiguredRMFailoverProxyProvider.java | 2 +- .../org/apache/hadoop/yarn/client/RMProxy.java | 45 +- .../RequestHedgingRMFailoverProxyProvider.java | 2 +- .../apache/hadoop/yarn/client/ServerProxy.java | 5 + .../yarn/client/api/impl/TimelineConnector.java | 9 +- .../hadoop/yarn/event/AsyncDispatcher.java | 12 +- .../apache/hadoop/yarn/event/Dispatcher.java | 10 - .../hadoop/yarn/event/EventDispatcher.java | 17 +- .../logaggregation/AggregatedLogFormat.java | 4 + .../yarn/util/ProcfsBasedProcessTree.java | 37 +- .../util/ResourceCalculatorProcessTree.java | 54 - .../yarn/util/WindowsBasedProcessTree.java | 12 - .../resource/DefaultResourceCalculator.java | 5 + .../resource/DominantResourceCalculator.java | 5 + .../yarn/util/resource/ResourceCalculator.java | 9 + .../hadoop/yarn/util/resource/Resources.java | 5 + .../org/apache/hadoop/yarn/webapp/WebApp.java | 6 +- .../hadoop/yarn/webapp/util/WebAppUtils.java | 17 +- .../src/main/resources/yarn-default.xml | 22 +- .../hadoop/yarn/api/TestPBImplRecords.java | 21 +- .../hadoop/yarn/client/TestClientRMProxy.java | 109 + .../client/api/impl/TestTimelineClient.java | 2 + .../hadoop/yarn/event/DrainDispatcher.java | 11 +- .../yarn/util/TestProcfsBasedProcessTree.java | 72 +- .../yarn/util/TestWindowsBasedProcessTree.java | 12 - .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 +- .../pom.xml | 16 +- ...pplicationHistoryManagerOnTimelineStore.java | 37 +- .../timeline/RollingLevelDBTimelineStore.java | 29 +- .../TimelineDelegationTokenIdentifierData.java | 4 +- ...pplicationHistoryManagerOnTimelineStore.java | 32 +- .../hadoop-yarn-server-common/pom.xml | 4 +- .../hadoop/yarn/server/api/ServerRMProxy.java | 4 +- .../hadoop-yarn-server-nodemanager/pom.xml | 4 +- .../nodemanager/CMgrCompletedAppsEvent.java | 2 +- .../CMgrCompletedContainersEvent.java | 2 +- .../server/nodemanager/ContainerExecutor.java | 4 +- .../nodemanager/DefaultContainerExecutor.java | 7 +- .../server/nodemanager/DeletionService.java | 468 +- .../nodemanager/LinuxContainerExecutor.java | 171 +- .../yarn/server/nodemanager/NMAuditLogger.java | 2 +- .../nodemanager/NodeHealthCheckerService.java | 54 +- .../yarn/server/nodemanager/NodeManager.java | 5 - .../server/nodemanager/NodeStatusUpdater.java | 6 + .../nodemanager/NodeStatusUpdaterImpl.java | 398 +- .../nodemanager/amrmproxy/AMRMProxyService.java | 63 +- .../nodemanager/api/impl/pb/NMProtoUtils.java | 110 + .../nodemanager/api/impl/pb/package-info.java | 25 + .../containermanager/ContainerLocalization.java | 21 - .../ContainerLocalizationImpl.java | 29 - .../containermanager/ContainerManager.java | 2 - .../containermanager/ContainerManagerImpl.java | 105 +- .../application/ApplicationImpl.java | 27 +- .../container/ContainerImpl.java | 9 +- .../recovery/DeletionTaskRecoveryInfo.java | 73 + .../deletion/recovery/package-info.java | 25 + .../deletion/task/DeletionTask.java | 258 + .../deletion/task/DeletionTaskType.java | 24 + .../deletion/task/FileDeletionTask.java | 202 + .../deletion/task/package-info.java | 25 + .../launcher/ContainerLaunch.java | 270 +- .../launcher/ContainerRelaunch.java | 9 + .../linux/resources/CGroupsHandler.java | 6 +- .../linux/resources/CGroupsHandlerImpl.java | 248 +- .../DelegatingLinuxContainerRuntime.java | 9 +- .../runtime/DockerLinuxContainerRuntime.java | 109 +- .../JavaSandboxLinuxContainerRuntime.java | 77 +- .../runtime/docker/DockerCommandExecutor.java | 191 + .../linux/runtime/docker/package-info.java | 26 + .../localizer/LocalResourcesTrackerImpl.java | 26 +- .../localizer/ResourceLocalizationService.java | 50 +- .../logaggregation/AppLogAggregatorImpl.java | 60 +- .../loghandler/NonAggregatingLogHandler.java | 7 +- .../monitor/ContainersMonitorImpl.java | 49 +- .../util/CgroupsLCEResourcesHandler.java | 32 +- .../impl/container-executor.c | 231 +- .../impl/container-executor.h | 28 +- .../container-executor/impl/get_executable.c | 2 +- .../test/test-container-executor.c | 34 + .../yarn_server_nodemanager_recovery.proto | 8 + .../nodemanager/DummyContainerManager.java | 6 +- .../TestDefaultContainerExecutor.java | 3 +- .../server/nodemanager/TestDeletionService.java | 57 +- .../nodemanager/TestLinuxContainerExecutor.java | 9 +- .../TestLinuxContainerExecutorWithMocks.java | 217 +- .../nodemanager/TestNodeHealthService.java | 11 +- .../nodemanager/TestNodeManagerReboot.java | 99 +- .../nodemanager/TestNodeManagerResync.java | 164 - .../amrmproxy/BaseAMRMProxyTest.java | 21 +- .../amrmproxy/TestAMRMProxyService.java | 65 + .../TestAMRMProxyTokenSecretManager.java | 81 + .../api/impl/pb/TestNMProtoUtils.java | 91 + .../BaseContainerManagerTest.java | 20 +- .../containermanager/TestContainerManager.java | 5 - .../TestContainerManagerRecovery.java | 56 +- .../containermanager/TestNMProxy.java | 30 +- .../deletion/task/FileDeletionMatcher.java | 84 + .../deletion/task/TestFileDeletionTask.java | 85 + .../launcher/TestContainerLaunch.java | 65 +- .../MockPrivilegedOperationCaptor.java | 68 + .../linux/resources/TestCGroupsHandlerImpl.java | 407 +- .../resources/TestResourceHandlerModule.java | 3 - .../runtime/TestDockerContainerRuntime.java | 42 +- .../TestJavaSandboxLinuxContainerRuntime.java | 126 +- .../docker/TestDockerCommandExecutor.java | 218 + .../runtime/docker/TestDockerLoadCommand.java | 48 + .../runtime/docker/TestDockerRunCommand.java | 63 + .../TestLocalResourcesTrackerImpl.java | 13 +- .../TestResourceLocalizationService.java | 106 +- .../TestAppLogAggregatorImpl.java | 15 +- .../TestLogAggregationService.java | 17 +- .../TestNonAggregatingLogHandler.java | 8 +- .../TestContainersMonitorResourceChange.java | 3 +- .../TestContainerSchedulerQueuing.java | 9 +- .../TestNMLeveldbStateStoreService.java | 2 +- .../util/TestCgroupsLCEResourcesHandler.java | 70 +- ...-container-executer-with-configuration-error | 20 + .../hadoop-yarn-server-resourcemanager/pom.xml | 5 +- ...ActiveStandbyElectorBasedElectorService.java | 5 +- .../server/resourcemanager/AdminService.java | 6 +- .../ApplicationMasterService.java | 2 +- .../resourcemanager/NodesListManager.java | 30 +- .../server/resourcemanager/RMAppManager.java | 21 +- .../server/resourcemanager/RMAuditLogger.java | 2 +- ...MCriticalThreadUncaughtExceptionHandler.java | 19 +- .../server/resourcemanager/RMFatalEvent.java | 67 +- .../resourcemanager/RMFatalEventType.java | 1 + .../server/resourcemanager/ResourceManager.java | 199 +- .../metrics/AbstractSystemMetricsPublisher.java | 2 +- .../monitor/SchedulingMonitor.java | 55 +- .../CapacitySchedulerPreemptionContext.java | 5 + .../CapacitySchedulerPreemptionUtils.java | 9 +- .../FifoIntraQueuePreemptionPlugin.java | 329 +- .../capacity/IntraQueueCandidatesSelector.java | 110 +- .../IntraQueuePreemptionComputePlugin.java | 10 +- .../ProportionalCapacityPreemptionPolicy.java | 42 +- ...QueuePriorityContainerCandidateSelector.java | 2 +- .../monitor/capacity/TempAppPerPartition.java | 6 +- .../monitor/capacity/TempQueuePerPartition.java | 14 + .../monitor/capacity/TempUserPerPartition.java | 88 + .../invariants/InvariantViolationException.java | 35 + .../monitor/invariants/InvariantsChecker.java | 96 + .../invariants/MetricsInvariantChecker.java | 195 + .../ReservationInvariantsChecker.java | 63 + .../monitor/invariants/package-info.java | 22 + .../resourcemanager/recovery/RMStateStore.java | 12 +- .../recovery/ZKRMStateStore.java | 506 +- .../reservation/InMemoryPlan.java | 11 + .../PeriodicRLESparseResourceAllocation.java | 167 + .../resourcemanager/reservation/PlanView.java | 9 + .../RLESparseResourceAllocation.java | 51 + .../planning/AlignedPlannerWithGreedy.java | 15 +- .../planning/GreedyReservationAgent.java | 13 +- .../reservation/planning/IterativePlanner.java | 196 +- .../reservation/planning/ReservationAgent.java | 23 +- .../planning/SimpleCapacityReplanner.java | 8 +- .../reservation/planning/StageAllocator.java | 10 +- .../planning/StageAllocatorGreedy.java | 4 +- .../planning/StageAllocatorGreedyRLE.java | 4 +- .../planning/StageAllocatorLowCostAligned.java | 279 +- .../planning/StageEarliestStart.java | 46 - .../planning/StageEarliestStartByDemand.java | 106 - .../StageEarliestStartByJobArrival.java | 39 - .../planning/StageExecutionInterval.java | 47 + .../StageExecutionIntervalByDemand.java | 144 + .../StageExecutionIntervalUnconstrained.java | 73 + .../server/resourcemanager/rmapp/RMAppImpl.java | 24 +- .../rmapp/attempt/RMAppAttemptImpl.java | 6 +- .../rmapp/attempt/RMAppAttemptMetrics.java | 2 +- .../scheduler/AbstractYarnScheduler.java | 2 +- .../resourcemanager/scheduler/NodeType.java | 12 +- .../resourcemanager/scheduler/QueueMetrics.java | 21 +- .../scheduler/capacity/AbstractCSQueue.java | 25 + .../scheduler/capacity/CSQueueMetrics.java | 4 +- .../CapacitySchedulerConfiguration.java | 8 + .../capacity/CapacitySchedulerQueueManager.java | 6 +- .../scheduler/capacity/LeafQueue.java | 11 +- .../scheduler/capacity/UsersManager.java | 11 +- .../allocator/AbstractContainerAllocator.java | 6 +- .../fair/AllocationFileLoaderService.java | 3 +- .../scheduler/fair/FSAppAttempt.java | 44 +- .../scheduler/fair/FSLeafQueue.java | 9 +- .../scheduler/fair/FSParentQueue.java | 4 +- .../resourcemanager/scheduler/fair/FSQueue.java | 22 +- .../scheduler/fair/FSQueueMetrics.java | 4 +- .../scheduler/fair/FSSchedulerNode.java | 8 +- .../scheduler/fair/FairScheduler.java | 27 +- .../scheduler/fair/FairSchedulerUtilities.java | 69 + .../scheduler/fair/QueueManager.java | 6 +- .../scheduler/fair/SchedulingPolicy.java | 38 +- .../fair/policies/FairSharePolicy.java | 2 +- .../resourcemanager/webapp/RMAppsBlock.java | 17 +- .../resourcemanager/webapp/RMWebAppUtil.java | 149 + .../yarn/server/resourcemanager/HATestUtil.java | 9 +- .../yarn/server/resourcemanager/MockRM.java | 10 +- .../server/resourcemanager/TestAppManager.java | 2 + .../resourcemanager/TestClientRMService.java | 25 + .../resourcemanager/TestRMDispatcher.java | 1 + .../resourcemanager/TestRMStoreCommands.java | 15 +- .../applicationsmanager/TestAMRestart.java | 140 +- .../monitor/TestSchedulingMonitor.java | 19 + ...alCapacityPreemptionPolicyMockFramework.java | 89 +- ...ionalCapacityPreemptionPolicyIntraQueue.java | 30 +- ...cityPreemptionPolicyIntraQueueUserLimit.java | 899 ++++ ...pacityPreemptionPolicyIntraQueueWithDRF.java | 178 + .../invariants/TestMetricsInvariantChecker.java | 99 + .../recovery/RMStateStoreTestBase.java | 10 +- .../recovery/TestFSRMStateStore.java | 1 + .../recovery/TestLeveldbRMStateStore.java | 1 + .../recovery/TestMemoryRMStateStore.java | 2 + .../recovery/TestZKRMStateStore.java | 573 +- .../reservation/ReservationSystemTestUtil.java | 14 + ...TestPeriodicRLESparseResourceAllocation.java | 142 + .../TestRLESparseResourceAllocation.java | 56 +- .../planning/TestAlignedPlanner.java | 417 +- .../planning/TestGreedyReservationAgent.java | 14 +- .../planning/TestSimpleCapacityReplanner.java | 4 +- .../scheduler/TestAbstractYarnScheduler.java | 2 +- .../capacity/TestCapacityScheduler.java | 15 +- .../scheduler/fair/TestFairScheduler.java | 15 + .../fair/TestFairSchedulerPreemption.java | 4 + .../fair/TestFairSchedulerUtilities.java | 67 + .../fair/policies/TestEmptyQueues.java | 6 +- .../src/test/resources/invariants.txt | 54 + .../pom.xml | 4 +- .../hadoop-yarn-server-tests/pom.xml | 4 +- .../hadoop/yarn/server/MiniYARNCluster.java | 22 +- .../server/TestContainerManagerSecurity.java | 14 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 34 +- .../pom.xml | 4 +- .../pom.xml | 4 +- .../pom.xml | 2 +- .../storage/TimelineSchemaCreator.java | 144 +- .../storage/application/ApplicationTable.java | 12 +- .../storage/entity/EntityTable.java | 12 +- .../hadoop-yarn-server-timelineservice/pom.xml | 4 +- .../collector/AppLevelTimelineCollector.java | 8 +- .../PerNodeTimelineCollectorsAuxService.java | 7 +- .../collector/TimelineCollector.java | 27 +- .../collector/TimelineCollectorManager.java | 6 +- .../reader/filter/TimelineFilterList.java | 2 +- .../storage/FileSystemTimelineReaderImpl.java | 129 +- .../collector/TestTimelineCollector.java | 95 +- .../hadoop-yarn-server-web-proxy/pom.xml | 4 +- .../server/webproxy/amfilter/AmIpFilter.java | 6 + .../server/webproxy/amfilter/TestAmFilter.java | 8 + .../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +- .../src/site/markdown/FairScheduler.md | 6 + .../src/site/markdown/ResourceManagerHA.md | 4 +- .../src/site/markdown/ResourceManagerRestart.md | 2 +- .../src/site/markdown/TimelineServiceV2.md | 5 +- .../hadoop-yarn/hadoop-yarn-ui/README.md | 64 + .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 156 +- .../hadoop-yarn-ui/src/main/webapp/.bowerrc | 5 +- .../src/main/webapp/WEB-INF/wro.xml | 9 + .../webapp/app/adapters/yarn-app-attempt.js | 4 +- .../webapp/app/adapters/yarn-app-flowrun.js | 33 + .../webapp/app/adapters/yarn-app-timeline.js | 35 + .../src/main/webapp/app/adapters/yarn-app.js | 2 +- .../main/webapp/app/adapters/yarn-container.js | 18 +- .../src/main/webapp/app/adapters/yarn-entity.js | 35 + .../webapp/app/adapters/yarn-flow-activity.js | 30 + .../webapp/app/adapters/yarn-flowrun-brief.js | 32 + .../main/webapp/app/adapters/yarn-flowrun.js | 38 + .../app/adapters/yarn-timeline-appattempt.js | 41 + .../app/adapters/yarn-timeline-container.js | 40 + .../webapp/app/components/em-table-html-cell.js | 23 + .../main/webapp/app/components/nodes-heatmap.js | 18 +- .../webapp/app/components/simple-bar-chart.js | 206 + .../webapp/app/components/sunburst-chart.js | 261 + .../main/webapp/app/components/timeline-view.js | 13 +- .../webapp/app/controllers/app-table-columns.js | 213 +- .../webapp/app/controllers/yarn-app-attempt.js | 32 +- .../webapp/app/controllers/yarn-app-attempts.js | 40 - .../src/main/webapp/app/controllers/yarn-app.js | 53 +- .../webapp/app/controllers/yarn-app/attempts.js | 24 + .../webapp/app/controllers/yarn-app/charts.js | 28 + .../webapp/app/controllers/yarn-app/info.js | 32 + .../webapp/app/controllers/yarn-apps/apps.js | 10 +- .../app/controllers/yarn-apps/services.js | 25 - .../app/controllers/yarn-flow-activity.js | 125 + .../main/webapp/app/controllers/yarn-flow.js | 36 + .../webapp/app/controllers/yarn-flow/info.js | 66 + .../webapp/app/controllers/yarn-flow/runs.js | 178 + .../main/webapp/app/controllers/yarn-flowrun.js | 50 + .../webapp/app/controllers/yarn-flowrun/info.js | 157 + .../app/controllers/yarn-flowrun/metrics.js | 126 + .../webapp/app/controllers/yarn-nodes/table.js | 8 + .../webapp/app/controllers/yarn-services.js | 89 +- .../src/main/webapp/app/mixins/app-attempt.js | 66 + .../main/webapp/app/models/yarn-app-attempt.js | 11 +- .../main/webapp/app/models/yarn-app-flowrun.js | 52 + .../main/webapp/app/models/yarn-app-timeline.js | 105 + .../src/main/webapp/app/models/yarn-app.js | 6 +- .../main/webapp/app/models/yarn-container.js | 9 +- .../src/main/webapp/app/models/yarn-entity.js | 26 + .../webapp/app/models/yarn-flow-activity.js | 28 + .../webapp/app/models/yarn-flowrun-brief.js | 70 + .../src/main/webapp/app/models/yarn-flowrun.js | 32 + .../app/models/yarn-timeline-appattempt.js | 143 + .../app/models/yarn-timeline-container.js | 63 + .../src/main/webapp/app/router.js | 21 +- .../main/webapp/app/routes/timeline-error.js | 29 + .../main/webapp/app/routes/yarn-app-attempt.js | 32 +- .../main/webapp/app/routes/yarn-app-attempts.js | 34 - .../src/main/webapp/app/routes/yarn-app.js | 35 +- .../main/webapp/app/routes/yarn-app/attempts.js | 37 + .../main/webapp/app/routes/yarn-app/charts.js | 53 + .../src/main/webapp/app/routes/yarn-app/info.js | 37 + .../webapp/app/routes/yarn-apps/services.js | 33 - .../webapp/app/routes/yarn-flow-activity.js | 37 + .../src/main/webapp/app/routes/yarn-flow.js | 53 + .../main/webapp/app/routes/yarn-flow/info.js | 22 + .../main/webapp/app/routes/yarn-flow/runs.js | 22 + .../webapp/app/routes/yarn-flowrun-metric.js | 107 + .../src/main/webapp/app/routes/yarn-flowrun.js | 58 + .../main/webapp/app/routes/yarn-flowrun/info.js | 22 + .../webapp/app/routes/yarn-flowrun/metrics.js | 22 + .../src/main/webapp/app/routes/yarn-services.js | 34 + .../webapp/app/serializers/yarn-app-flowrun.js | 64 + .../webapp/app/serializers/yarn-app-timeline.js | 76 + .../src/main/webapp/app/serializers/yarn-app.js | 3 +- .../main/webapp/app/serializers/yarn-entity.js | 51 + .../app/serializers/yarn-flow-activity.js | 54 + .../app/serializers/yarn-flowrun-brief.js | 64 + .../main/webapp/app/serializers/yarn-flowrun.js | 63 + .../main/webapp/app/serializers/yarn-queue.js | 22 +- .../app/serializers/yarn-timeline-appattempt.js | 69 + .../app/serializers/yarn-timeline-container.js | 71 + .../src/main/webapp/app/styles/app.css | 92 + .../main/webapp/app/templates/application.hbs | 19 +- .../templates/components/app-attempt-table.hbs | 8 +- .../app/templates/components/app-table.hbs | 6 +- .../templates/components/container-table.hbs | 2 +- .../templates/components/em-table-html-cell.hbs | 23 + .../app/templates/components/timeline-view.hbs | 8 +- .../webapp/app/templates/timeline-error.hbs | 22 + .../webapp/app/templates/yarn-app-attempt.hbs | 46 +- .../webapp/app/templates/yarn-app-attempts.hbs | 55 - .../src/main/webapp/app/templates/yarn-app.hbs | 232 +- .../webapp/app/templates/yarn-app/attempts.hbs | 29 + .../webapp/app/templates/yarn-app/charts.hbs | 43 + .../main/webapp/app/templates/yarn-app/info.hbs | 167 + .../webapp/app/templates/yarn-app/loading.hbs | 23 + .../src/main/webapp/app/templates/yarn-apps.hbs | 11 +- .../webapp/app/templates/yarn-apps/apps.hbs | 2 +- .../webapp/app/templates/yarn-apps/services.hbs | 25 - .../webapp/app/templates/yarn-flow-activity.hbs | 45 + .../src/main/webapp/app/templates/yarn-flow.hbs | 49 + .../webapp/app/templates/yarn-flow/info.hbs | 53 + .../webapp/app/templates/yarn-flow/runs.hbs | 131 + .../app/templates/yarn-flowrun-metric.hbs | 34 + .../main/webapp/app/templates/yarn-flowrun.hbs | 49 + .../webapp/app/templates/yarn-flowrun/info.hbs | 128 + .../app/templates/yarn-flowrun/metrics.hbs | 34 + .../webapp/app/templates/yarn-nodes/table.hbs | 2 +- .../main/webapp/app/templates/yarn-services.hbs | 85 + .../src/main/webapp/app/utils/converter.js | 44 +- .../src/main/webapp/app/utils/error-utils.js | 58 + .../src/main/webapp/bower-shrinkwrap.json | 66 + .../src/main/webapp/config/default-config.js | 3 +- .../src/main/webapp/ember-cli-build.js | 16 +- .../hadoop-yarn-ui/src/main/webapp/package.json | 11 +- .../components/em-table-html-cell-test.js | 43 + .../components/simple-bar-chart-test.js | 43 + .../unit/adapters/yarn-app-timeline-test.js | 30 + .../adapters/yarn-timeline-appattempt-test.js | 30 + .../adapters/yarn-timeline-container-test.js | 30 + .../unit/controllers/yarn-app-attempts-test.js | 30 - .../unit/controllers/yarn-app/attempts-test.js | 30 + .../unit/controllers/yarn-app/charts-test.js | 30 + .../unit/controllers/yarn-app/info-test.js | 30 + .../unit/controllers/yarn-apps/services-test.js | 30 - .../unit/controllers/yarn-flow/info-test.js | 30 + .../unit/controllers/yarn-flow/runs-test.js | 30 + .../unit/controllers/yarn-flowrun/info-test.js | 30 + .../controllers/yarn-flowrun/metrics-test.js | 30 + .../tests/unit/mixins/app-attempt-test.js | 30 + .../webapp/tests/unit/mixins/charts-test.js | 30 - .../tests/unit/models/yarn-app-timeline-test.js | 30 + .../models/yarn-timeline-appattempt-test.js | 30 + .../unit/models/yarn-timeline-container-test.js | 30 + .../tests/unit/routes/timeline-error-test.js | 29 + .../tests/unit/routes/yarn-app-attempts-test.js | 29 - .../tests/unit/routes/yarn-app/attempts-test.js | 29 + .../tests/unit/routes/yarn-app/charts-test.js | 29 + .../tests/unit/routes/yarn-app/info-test.js | 29 + .../tests/unit/routes/yarn-flow/info-test.js | 29 + .../tests/unit/routes/yarn-flow/runs-test.js | 29 + .../tests/unit/routes/yarn-flowrun/info-test.js | 29 + .../unit/routes/yarn-flowrun/metrics-test.js | 29 + .../tests/unit/routes/yarn-services-test.js | 29 + .../unit/serializers/yarn-app-timeline-test.js | 33 + .../yarn-timeline-appattempt-test.js | 33 + .../serializers/yarn-timeline-container-test.js | 33 + .../hadoop-yarn-ui/src/main/webapp/yarn.lock | 4983 ++++++++++++++++++ hadoop-yarn-project/hadoop-yarn/pom.xml | 4 +- hadoop-yarn-project/pom.xml | 4 +- pom.xml | 2 +- 1148 files changed, 45884 insertions(+), 10731 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java ---------------------------------------------------------------------- diff --cc hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java index 17d2404,92e2135..5b2ba65 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java @@@ -705,26 -748,13 +748,27 @@@ public abstract class FileSystemContrac /** * This a sanity check to make sure that any filesystem's handling of - * renames doesn't cause any regressions + * renames empty dirs doesn't cause any regressions. + */ + public void testRenameEmptyToDirWithSamePrefixAllowed() throws Throwable { + if (!renameSupported()) return; + Path parentdir = path("testRenameEmptyToDirWithSamePrefixAllowed"); + fs.mkdirs(parentdir); + Path dest = path("testRenameEmptyToDirWithSamePrefixAllowedDest"); + rename(parentdir, dest, true, false, true); + } + + /** + * This a sanity check to make sure that any filesystem's handling of + * renames non-empty dirs doesn't cause any regressions. */ + @Test public void testRenameToDirWithSamePrefixAllowed() throws Throwable { - if (!renameSupported()) return; + assumeTrue(renameSupported()); final Path parentdir = path("testRenameToDirWithSamePrefixAllowed"); fs.mkdirs(parentdir); + // Before renaming, we create one file under the source parent directory + createFile(new Path(parentdir, "mychild")); final Path dest = path("testRenameToDirWithSamePrefixAllowedDest"); rename(parentdir, dest, true, false, true); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index e14e341,9e15b3f..0fb1197 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@@ -313,123 -320,4 +325,114 @@@ public final class Constants @InterfaceAudience.Private public static final int MAX_MULTIPART_COUNT = 10000; + /** + * Classname of the S3A-specific output committer factory. This + * is what must be declared when attempting to use + */ + @InterfaceStability.Unstable + public static final String S3A_OUTPUT_COMMITTER_FACTORY = + "org.apache.hadoop.fs.s3a.commit.S3AOutputCommitterFactory"; + + /* Constants. */ + public static final String S3_METADATA_STORE_IMPL = + "fs.s3a.metadatastore.impl"; + + /** Minimum period of time (in milliseconds) to keep metadata (may only be + * applied when a prune command is manually run). + */ + @InterfaceStability.Unstable + public static final String S3GUARD_CLI_PRUNE_AGE = + "fs.s3a.s3guard.cli.prune.age"; + + /** + * The region of the DynamoDB service. + * + * This config has no default value. If the user does not set this, the + * S3Guard will operate table in the associated S3 bucket region. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_REGION_KEY = + "fs.s3a.s3guard.ddb.region"; + + /** + * The DynamoDB table name to use. + * + * This config has no default value. If the user does not set this, the + * S3Guard implementation will use the respective S3 bucket name. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_TABLE_NAME_KEY = + "fs.s3a.s3guard.ddb.table"; + + /** + * Whether to create the DynamoDB table if the table does not exist. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_TABLE_CREATE_KEY = + "fs.s3a.s3guard.ddb.table.create"; + + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_TABLE_CAPACITY_READ_KEY = + "fs.s3a.s3guard.ddb.table.capacity.read"; + public static final long S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT = 500; + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY = + "fs.s3a.s3guard.ddb.table.capacity.write"; + public static final long S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT = 100; + + /** + * The maximum put or delete requests per BatchWriteItem request. + * + * Refer to Amazon API reference for this limit. + */ + public static final int S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT = 25; + + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_MAX_RETRIES = + "fs.s3a.s3guard.ddb.max.retries"; + /** + * Max retries on batched DynamoDB operations before giving up and + * throwing an IOException. Default is {@value}. See core-default.xml for + * more detail. + */ + public static final int S3GUARD_DDB_MAX_RETRIES_DEFAULT = 9; + + /** + * Period of time (in milliseconds) to sleep between batches of writes. + * Currently only applies to prune operations, as they are naturally a + * lower priority than other operations. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY = + "fs.s3a.s3guard.ddb.background.sleep"; + public static final int S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_DEFAULT = 25; + + /** + * V1 committer. + */ + @InterfaceStability.Unstable + public static final String S3A_OUTPUT_COMMITTER_MRV1 = + "org.apache.hadoop.fs.s3a.commit.S3OutputCommitterMRv1"; + + /** + * The default "Null" metadata store: {@value}. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_METASTORE_NULL + = "org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore"; + + /** + * Use Local memory for the metadata: {@value}. + * This is not coherent across processes and must be used for testing only. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_METASTORE_LOCAL + = "org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore"; + + /** + * Use DynamoDB for the metadata: {@value}. + */ + @InterfaceStability.Unstable + public static final String S3GUARD_METASTORE_DYNAMO + = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore"; - - @InterfaceAudience.Private - public static final String SSE_C_NO_KEY_ERROR = S3AEncryptionMethods.SSE_C - .getMethod() +" is enabled and no encryption key is provided."; - - - @InterfaceAudience.Private - public static final String SSE_S3_WITH_KEY_ERROR = S3AEncryptionMethods.SSE_S3 - .getMethod() +" is configured and an " + "encryption key is provided"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 7de94cc,872dd5f..4eb94ad --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@@ -1675,65 -1553,45 +1702,67 @@@ public class S3AFileSystem extends File * @throws IOException other IO problems * @throws AmazonClientException on failures inside the AWS SDK */ - private boolean innerMkdirs(Path f, FsPermission permission) + private boolean innerMkdirs(Path p, FsPermission permission) throws IOException, FileAlreadyExistsException, AmazonClientException { + boolean createOnS3 = false; + Path f = qualify(p); LOG.debug("Making directory: {}", f); incrementStatistic(INVOCATION_MKDIRS); - FileStatus fileStatus; - try { - fileStatus = getFileStatus(f); + List<Path> metadataStoreDirs = null; + if (hasMetadataStore()) { + metadataStoreDirs = new ArrayList<>(); + } - if (fileStatus.isDirectory()) { - return true; - } else { - throw new FileAlreadyExistsException("Path is a file: " + f); + DirectoryStatus status = checkPathForDirectory(f); + if (status == DirectoryStatus.DOES_NOT_EXIST) { + createOnS3 = true; + if (metadataStoreDirs != null) { + metadataStoreDirs.add(f); } - } catch (FileNotFoundException e) { - Path fPart = f.getParent(); - do { - try { - fileStatus = getFileStatus(fPart); - if (fileStatus.isDirectory()) { - break; - } - if (fileStatus.isFile()) { - throw new FileAlreadyExistsException(String.format( - "Can't make directory for path '%s' since it is a file.", - fPart)); - } - } catch (FileNotFoundException fnfe) { - instrumentation.errorIgnored(); + } else if (status == DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY) { + if (metadataStoreDirs != null) { + metadataStoreDirs.add(f); + } + } else if (status == DirectoryStatus + .EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE) { + return true; + } else if (status == DirectoryStatus.EXISTS_AND_IS_FILE) { + throw new FileAlreadyExistsException("Path is a file: " + f); + } + + // Walk path to root, ensuring closest ancestor is a directory, not file + Path fPart = f.getParent(); + do { + status = checkPathForDirectory(fPart); + // The fake directory on S3 may not be visible immediately, but + // only the leaf node has a fake directory on S3 so we can treat both + // cases the same and just create a metadata store entry. + if (status == DirectoryStatus.DOES_NOT_EXIST || status == + DirectoryStatus.EXISTS_AND_IS_DIRECTORY_ON_S3_ONLY) { + if (metadataStoreDirs != null) { + metadataStoreDirs.add(fPart); } - fPart = fPart.getParent(); - } while (fPart != null); + } else if (status == DirectoryStatus + .EXISTS_AND_IS_DIRECTORY_ON_METADATASTORE) { + // do nothing - just break out of the loop, make whatever child + // directories are needed on the metadata store, and return the + // result. + break; + } else if (status == DirectoryStatus.EXISTS_AND_IS_FILE) { + throw new FileAlreadyExistsException("Path is a file: " + f); + } + fPart = fPart.getParent(); + } while (fPart != null); + if (createOnS3) { String key = pathToKey(f); createFakeDirectory(key); - // this is complicated because getParent(a/b/c/) returns a/b/c, but - // we want a/b. See HADOOP-14428 for more details. - deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent()); - return true; } + S3Guard.makeDirsOrdered(metadataStore, metadataStoreDirs, username); - deleteUnnecessaryFakeDirectories(f.getParent()); ++ // this is complicated because getParent(a/b/c/) returns a/b/c, but ++ // we want a/b. See HADOOP-14428 for more details. ++ deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent()); + return true; } /** @@@ -2025,16 -1756,33 +2051,34 @@@ // Since we have a local file, we don't need to stream into a temporary file LocalFileSystem local = getLocal(getConf()); File srcfile = local.pathToFile(src); + if (!srcfile.exists()) { + throw new FileNotFoundException("No file: " + src); + } + if (!srcfile.isFile()) { + throw new FileNotFoundException("Not a file: " + src); + } + try { + FileStatus status = getFileStatus(dst); + if (!status.isFile()) { + throw new FileAlreadyExistsException(dst + " exists and is not a file"); + } + if (!overwrite) { + throw new FileAlreadyExistsException(dst + " already exists"); + } + } catch (FileNotFoundException e) { + // no destination, all is well + } + final String key = pathToKey(dst); final ObjectMetadata om = newObjectMetadata(srcfile.length()); PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, srcfile); - Upload up = putObject(putObjectRequest); + UploadInfo info = putObject(putObjectRequest); + Upload upload = info.getUpload(); ProgressableProgressListener listener = new ProgressableProgressListener( - this, key, up, null); - up.addProgressListener(listener); + this, key, upload, null); + upload.addProgressListener(listener); try { - up.waitForUploadResult(); + upload.waitForUploadResult(); } catch (InterruptedException e) { throw new InterruptedIOException("Interrupted copying " + src + " to " + dst + ", cancelling"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java index 0000000,71776ac..7dc286d mode 000000,100644..100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java @@@ -1,0 -1,153 +1,154 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.hadoop.fs.s3a; + + import java.io.File; + import java.io.FileNotFoundException; + import java.io.IOException; + import java.nio.charset.Charset; + + import org.junit.Test; + + import org.apache.commons.io.Charsets; + import org.apache.commons.io.FileUtils; + import org.apache.commons.io.IOUtils; + import org.apache.hadoop.fs.FileAlreadyExistsException; ++import org.apache.hadoop.fs.FileStatus; + import org.apache.hadoop.fs.Path; + + import static org.apache.hadoop.test.LambdaTestUtils.intercept; + + /** + * Test {@link S3AFileSystem#copyFromLocalFile(boolean, boolean, Path, Path)}. + */ + public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase { + private static final Charset ASCII = Charsets.US_ASCII; + + private File file; + + @Override + public void teardown() throws Exception { + super.teardown(); + if (file != null) { + file.delete(); + } + } + + @Test + public void testCopyEmptyFile() throws Throwable { + file = File.createTempFile("test", ".txt"); + Path dest = upload(file, true); + assertPathExists("uploaded file", dest); + } + + @Test + public void testCopyFile() throws Throwable { + String message = "hello"; + file = createTempFile(message); + Path dest = upload(file, true); + assertPathExists("uploaded file not found", dest); + S3AFileSystem fs = getFileSystem(); - S3AFileStatus status = fs.getFileStatus(dest); ++ FileStatus status = fs.getFileStatus(dest); + assertEquals("File length of " + status, + message.getBytes(ASCII).length, status.getLen()); + assertFileTextEquals(dest, message); + } + + public void assertFileTextEquals(Path path, String expected) + throws IOException { + assertEquals("Wrong data in " + path, + expected, IOUtils.toString(getFileSystem().open(path), ASCII)); + } + + @Test + public void testCopyFileNoOverwrite() throws Throwable { + file = createTempFile("hello"); + Path dest = upload(file, true); + intercept(FileAlreadyExistsException.class, + () -> upload(file, false)); + } + + @Test + public void testCopyFileOverwrite() throws Throwable { + file = createTempFile("hello"); + Path dest = upload(file, true); + String updated = "updated"; + FileUtils.write(file, updated, ASCII); + upload(file, true); + assertFileTextEquals(dest, updated); + } + + @Test + public void testCopyFileNoOverwriteDirectory() throws Throwable { + file = createTempFile("hello"); + Path dest = upload(file, true); + S3AFileSystem fs = getFileSystem(); + fs.delete(dest, false); + fs.mkdirs(dest); + intercept(FileAlreadyExistsException.class, + () -> upload(file, true)); + } + + @Test + public void testCopyMissingFile() throws Throwable { + file = File.createTempFile("test", ".txt"); + file.delete(); + // first upload to create + intercept(FileNotFoundException.class, "No file", + () -> upload(file, true)); + } + + @Test + public void testCopyDirectoryFile() throws Throwable { + file = File.createTempFile("test", ".txt"); + // first upload to create + intercept(FileNotFoundException.class, "Not a file", + () -> upload(file.getParentFile(), true)); + } + + + @Test + public void testLocalFilesOnly() throws Throwable { + Path dst = path("testLocalFilesOnly"); + intercept(IllegalArgumentException.class, + () -> { + getFileSystem().copyFromLocalFile(false, true, dst, dst); + return "copy successful"; + }); + } + + public Path upload(File srcFile, boolean overwrite) throws IOException { + Path src = new Path(srcFile.toURI()); + Path dst = path(srcFile.getName()); + getFileSystem().copyFromLocalFile(false, overwrite, src, dst); + return dst; + } + + /** + * Create a temp file with some text. + * @param text text for the file + * @return the file + * @throws IOException on a failure + */ + public File createTempFile(String text) throws IOException { + File f = File.createTempFile("test", ".txt"); + FileUtils.write(f, text, ASCII); + return f; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f02de2e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java index 6a0f878,1b49d07..2d47bb3 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java @@@ -26,9 -27,9 +27,11 @@@ import org.slf4j.LoggerFactory import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystemContractBaseTest; import org.apache.hadoop.fs.Path; + import static org.junit.Assume.*; + import static org.junit.Assert.*; +import java.io.IOException; + /** * Tests a live S3 system. If your keys and bucket aren't specified, all tests * are marked as passed. --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
