Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db22affd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db22affd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db22affd Branch: refs/heads/HDFS-7240 Commit: db22affd79b1a71bc25c5a2e133f6257530e855c Parents: b5f772a 03a9343 Author: Anu Engineer <[email protected]> Authored: Fri Aug 19 13:32:24 2016 -0700 Committer: Anu Engineer <[email protected]> Committed: Fri Aug 19 13:32:24 2016 -0700 ---------------------------------------------------------------------- LICENSE.txt | 209 + NOTICE.txt | 172 + dev-support/docker/Dockerfile | 9 +- hadoop-assemblies/pom.xml | 4 +- hadoop-build-tools/pom.xml | 46 +- hadoop-client/pom.xml | 4 +- .../hadoop-annotations/pom.xml | 4 +- .../hadoop-auth-examples/pom.xml | 4 +- hadoop-common-project/hadoop-auth/pom.xml | 4 +- .../authentication/util/KerberosName.java | 24 +- .../util/ZKSignerSecretProvider.java | 5 +- .../dev-support/findbugsExcludeFile.xml | 13 + hadoop-common-project/hadoop-common/pom.xml | 18 +- .../hadoop-common/src/main/bin/hadoop | 6 +- .../src/main/bin/hadoop-config.cmd | 4 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 4 + .../src/main/bin/hadoop-daemons.sh | 6 +- .../src/main/bin/hadoop-functions.sh | 75 +- .../hadoop-common/src/main/bin/rcc | 41 - .../hadoop-common/src/main/bin/slaves.sh | 60 - .../hadoop-common/src/main/bin/workers.sh | 60 + .../hadoop-common/src/main/conf/hadoop-env.sh | 51 +- .../main/conf/hadoop-user-functions.sh.example | 4 +- .../src/main/conf/log4j.properties | 17 +- .../hadoop-common/src/main/conf/workers | 1 + .../org/apache/hadoop/conf/ConfigRedactor.java | 84 + .../org/apache/hadoop/conf/Configuration.java | 5 +- .../apache/hadoop/conf/ReconfigurableBase.java | 13 +- .../apache/hadoop/crypto/key/KeyProvider.java | 13 +- .../crypto/key/KeyProviderCryptoExtension.java | 31 +- .../crypto/key/kms/KMSClientProvider.java | 29 +- .../key/kms/LoadBalancingKMSClientProvider.java | 5 +- .../apache/hadoop/fs/CachingGetSpaceUsed.java | 28 +- .../hadoop/fs/CommonConfigurationKeys.java | 15 +- .../fs/CommonConfigurationKeysPublic.java | 489 +- .../src/main/java/org/apache/hadoop/fs/DU.java | 10 +- .../apache/hadoop/fs/DelegateToFileSystem.java | 5 +- .../hadoop/fs/EmptyStorageStatistics.java | 7 + .../org/apache/hadoop/fs/FSInputStream.java | 5 +- .../java/org/apache/hadoop/fs/FileContext.java | 20 +- .../java/org/apache/hadoop/fs/FileSystem.java | 61 +- .../hadoop/fs/FileSystemStorageStatistics.java | 21 +- .../java/org/apache/hadoop/fs/FileUtil.java | 73 +- .../java/org/apache/hadoop/fs/GetSpaceUsed.java | 25 + .../java/org/apache/hadoop/fs/GlobExpander.java | 2 +- .../java/org/apache/hadoop/fs/GlobFilter.java | 2 +- .../hadoop/fs/GlobalStorageStatistics.java | 18 +- .../main/java/org/apache/hadoop/fs/Path.java | 181 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 12 +- .../org/apache/hadoop/fs/StorageStatistics.java | 61 +- .../hadoop/fs/UnionStorageStatistics.java | 26 +- .../apache/hadoop/fs/WindowsGetSpaceUsed.java | 6 +- .../hadoop/fs/permission/PermissionParser.java | 23 +- .../org/apache/hadoop/fs/shell/AclCommands.java | 18 +- .../org/apache/hadoop/fs/shell/Display.java | 8 +- .../java/org/apache/hadoop/fs/shell/Test.java | 80 +- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 23 +- .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 81 +- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 +- .../apache/hadoop/ha/ActiveStandbyElector.java | 19 +- .../java/org/apache/hadoop/ha/StreamPumper.java | 4 +- .../org/apache/hadoop/http/HtmlQuoting.java | 42 +- .../org/apache/hadoop/http/HttpServer2.java | 143 +- .../apache/hadoop/io/DefaultStringifier.java | 4 +- .../apache/hadoop/io/ElasticByteBufferPool.java | 1 + .../org/apache/hadoop/io/SecureIOUtils.java | 3 +- .../java/org/apache/hadoop/io/SequenceFile.java | 4 +- .../apache/hadoop/io/compress/BZip2Codec.java | 6 +- .../io/erasurecode/codec/HHXORErasureCodec.java | 45 + .../rawcoder/AbstractNativeRawDecoder.java | 84 + .../rawcoder/AbstractNativeRawEncoder.java | 81 + .../rawcoder/ByteArrayDecodingState.java | 23 + .../rawcoder/ByteArrayEncodingState.java | 23 + .../rawcoder/ByteBufferDecodingState.java | 14 +- .../rawcoder/ByteBufferEncodingState.java | 10 + .../io/erasurecode/rawcoder/CoderUtil.java | 12 - .../rawcoder/NativeRSRawDecoder.java | 66 + .../rawcoder/NativeRSRawEncoder.java | 65 + .../NativeRSRawErasureCoderFactory.java | 39 + .../rawcoder/RSRawDecoderLegacy.java | 56 +- .../rawcoder/RawErasureCoderFactory.java | 4 +- .../erasurecode/rawcoder/util/GaloisField.java | 4 +- .../hadoop/io/file/tfile/TFileDumper.java | 4 +- .../hadoop/io/retry/AsyncCallHandler.java | 104 +- .../org/apache/hadoop/io/retry/CallReturn.java | 3 + .../hadoop/io/retry/RetryInvocationHandler.java | 194 +- .../org/apache/hadoop/io/retry/RetryPolicy.java | 1 + .../apache/hadoop/io/retry/package-info.java | 22 + .../org/apache/hadoop/io/retry/package.html | 48 - .../org/apache/hadoop/ipc/CallQueueManager.java | 4 +- .../org/apache/hadoop/ipc/CallerContext.java | 4 +- .../main/java/org/apache/hadoop/ipc/Client.java | 15 +- .../apache/hadoop/ipc/DecayRpcScheduler.java | 17 +- .../org/apache/hadoop/ipc/FairCallQueue.java | 34 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 46 +- .../org/apache/hadoop/ipc/ResponseBuffer.java | 98 + .../org/apache/hadoop/ipc/RpcConstants.java | 6 +- .../java/org/apache/hadoop/ipc/RpcWritable.java | 184 + .../main/java/org/apache/hadoop/ipc/Server.java | 214 +- .../org/apache/hadoop/ipc/package-info.java | 4 + .../java/org/apache/hadoop/ipc/package.html | 23 - .../org/apache/hadoop/jmx/JMXJsonServlet.java | 11 +- .../metrics2/impl/MetricsSourceAdapter.java | 33 +- .../hadoop/metrics2/lib/MutableQuantiles.java | 13 +- .../hadoop/metrics2/sink/GraphiteSink.java | 5 +- .../sink/ganglia/AbstractGangliaSink.java | 4 +- .../org/apache/hadoop/net/TableMapping.java | 6 +- .../org/apache/hadoop/security/Credentials.java | 26 +- .../java/org/apache/hadoop/security/Groups.java | 143 +- .../hadoop/security/LdapGroupsMapping.java | 16 +- .../apache/hadoop/security/ProviderUtils.java | 2 - .../apache/hadoop/security/SaslRpcClient.java | 19 +- .../apache/hadoop/security/SaslRpcServer.java | 8 +- .../hadoop/security/ShellBasedIdMapping.java | 4 +- .../hadoop/security/UserGroupInformation.java | 61 +- .../alias/AbstractJavaKeyStoreProvider.java | 4 +- .../security/alias/CredentialProvider.java | 6 +- .../alias/CredentialProviderFactory.java | 6 +- .../hadoop/security/alias/UserProvider.java | 4 +- .../security/authorize/AccessControlList.java | 2 +- .../security/ssl/ReloadingX509TrustManager.java | 14 +- .../ssl/SslSelectChannelConnectorSecure.java | 58 + .../security/ssl/SslSocketConnectorSecure.java | 58 - .../org/apache/hadoop/security/token/Token.java | 37 +- .../ZKDelegationTokenSecretManager.java | 5 +- .../DelegationTokenAuthenticationHandler.java | 10 +- .../web/DelegationTokenAuthenticator.java | 77 +- .../delegation/web/DelegationTokenManager.java | 7 + .../org/apache/hadoop/tracing/TraceAdmin.java | 2 - .../hadoop/util/ApplicationClassLoader.java | 20 +- .../apache/hadoop/util/AutoCloseableLock.java | 122 + .../apache/hadoop/util/BasicDiskValidator.java | 34 + .../org/apache/hadoop/util/DataChecksum.java | 4 +- .../org/apache/hadoop/util/DiskValidator.java | 41 + .../hadoop/util/DiskValidatorFactory.java | 88 + .../org/apache/hadoop/util/FileBasedIPList.java | 4 +- .../hadoop/util/GenericOptionsParser.java | 134 +- .../org/apache/hadoop/util/HostsFileReader.java | 115 +- .../util/InvalidChecksumSizeException.java | 32 + .../java/org/apache/hadoop/util/LineReader.java | 5 +- .../java/org/apache/hadoop/util/RunJar.java | 6 +- .../main/java/org/apache/hadoop/util/Shell.java | 110 +- .../org/apache/hadoop/util/StringUtils.java | 35 +- .../apache/hadoop/util/bloom/BloomFilter.java | 2 +- .../hadoop-common/src/main/java/overview.html | 46 +- .../apache/hadoop/io/erasurecode/jni_common.c | 34 +- ...he_hadoop_io_erasurecode_ErasureCodeNative.h | 29 - ...io_erasurecode_rawcoder_NativeRSRawDecoder.h | 37 - ...io_erasurecode_rawcoder_NativeRSRawEncoder.h | 37 - .../src/main/resources/core-default.xml | 205 +- .../src/site/markdown/Benchmarking.md | 106 + .../src/site/markdown/ClusterSetup.md | 18 +- .../src/site/markdown/CommandsManual.md | 6 +- .../src/site/markdown/Compatibility.md | 1 + .../src/site/markdown/CredentialProviderAPI.md | 2 +- .../src/site/markdown/FileSystemShell.md | 3 + .../src/site/markdown/GroupsMapping.md | 10 +- .../hadoop-common/src/site/markdown/Metrics.md | 1 + .../src/site/markdown/RackAwareness.md | 4 +- .../src/site/markdown/UnixShellGuide.md | 8 +- .../src/site/markdown/filesystem/extending.md | 2 + .../src/site/markdown/filesystem/filesystem.md | 458 +- .../markdown/filesystem/fsdatainputstream.md | 9 +- .../site/markdown/filesystem/introduction.md | 4 +- .../src/site/markdown/filesystem/model.md | 68 +- .../src/site/markdown/filesystem/notation.md | 2 + .../src/site/markdown/filesystem/testing.md | 2 + .../conf/TestCommonConfigurationFields.java | 7 +- .../apache/hadoop/conf/TestConfigRedactor.java | 72 + .../hadoop/crypto/key/TestKeyProvider.java | 28 +- .../key/TestKeyProviderCryptoExtension.java | 208 + .../hadoop/fs/FileContextPermissionBase.java | 16 +- .../hadoop/fs/FileSystemContractBaseTest.java | 2 +- .../test/java/org/apache/hadoop/fs/TestDU.java | 10 +- .../hadoop/fs/TestDelegateToFsCheckPath.java | 126 + .../fs/TestFileSystemStorageStatistics.java | 134 + .../java/org/apache/hadoop/fs/TestFileUtil.java | 71 +- .../org/apache/hadoop/fs/TestFsShellCopy.java | 10 +- .../apache/hadoop/fs/TestLocalDirAllocator.java | 20 +- .../apache/hadoop/fs/TestLocalFileSystem.java | 8 +- .../fs/TestLocalFileSystemPermission.java | 119 +- .../hadoop/fs/TestLocal_S3FileContextURI.java | 38 - .../java/org/apache/hadoop/fs/TestPath.java | 24 +- .../hadoop/fs/TestS3_LocalFileContextURI.java | 38 - .../apache/hadoop/fs/TestSymlinkLocalFS.java | 18 +- .../fs/TestSymlinkLocalFSFileContext.java | 5 +- .../hadoop/fs/TestSymlinkLocalFSFileSystem.java | 5 +- .../AbstractContractGetFileStatusTest.java | 586 +- .../AbstractContractRootDirectoryTest.java | 44 +- .../fs/contract/AbstractContractSeekTest.java | 11 +- .../fs/contract/AbstractFSContractTestBase.java | 63 +- .../hadoop/fs/contract/ContractTestUtils.java | 195 +- .../hadoop/fs/sftp/TestSFTPFileSystem.java | 5 +- .../apache/hadoop/fs/shell/TestAclCommands.java | 34 +- .../apache/hadoop/fs/shell/TestPathData.java | 9 +- .../apache/hadoop/fs/shell/package-info.java | 26 - .../fs/viewfs/TestChRootedFileSystem.java | 65 + .../fs/viewfs/ViewFileSystemBaseTest.java | 54 +- .../hadoop/ha/TestActiveStandbyElector.java | 12 +- .../hadoop/http/HttpServerFunctionalTest.java | 19 + .../org/apache/hadoop/http/TestHttpServer.java | 68 +- .../codec/TestHHXORErasureCodec.java | 40 + .../rawcoder/RawErasureCoderBenchmark.java | 408 + .../io/erasurecode/rawcoder/TestCoderUtil.java | 124 + .../rawcoder/TestNativeRSRawCoder.java | 121 + .../rawcoder/TestRSRawCoderInteroperable1.java | 38 + .../rawcoder/TestRSRawCoderInteroperable2.java | 38 + .../rawcoder/TestRawErasureCoderBenchmark.java | 65 + .../apache/hadoop/io/nativeio/TestNativeIO.java | 51 +- .../hadoop/io/retry/TestDefaultRetryPolicy.java | 14 +- .../org/apache/hadoop/ipc/TestAsyncIPC.java | 4 +- .../apache/hadoop/ipc/TestCallQueueManager.java | 4 +- .../hadoop/ipc/TestDecayRpcScheduler.java | 30 +- .../apache/hadoop/ipc/TestFairCallQueue.java | 58 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 6 +- .../apache/hadoop/ipc/TestResponseBuffer.java | 87 + .../org/apache/hadoop/ipc/TestRpcWritable.java | 129 + .../java/org/apache/hadoop/ipc/TestSaslRPC.java | 12 +- .../apache/hadoop/ipc/TestSocketFactory.java | 201 +- .../apache/hadoop/jmx/TestJMXJsonServlet.java | 13 + .../metrics2/impl/TestGangliaMetrics.java | 63 +- .../java/org/apache/hadoop/net/TestDNS.java | 5 +- .../hadoop/security/TestGroupsCaching.java | 269 +- .../hadoop/security/TestLdapGroupsMapping.java | 21 + .../security/TestShellBasedIdMapping.java | 11 +- .../security/TestUserGroupInformation.java | 33 +- .../ssl/TestReloadingX509TrustManager.java | 63 +- ...tionTokenAuthenticationHandlerWithMocks.java | 102 +- .../delegation/web/TestWebDelegationToken.java | 114 +- .../apache/hadoop/test/GenericTestUtils.java | 31 +- .../org/apache/hadoop/test/HadoopTestBase.java | 67 + .../apache/hadoop/test/PlatformAssumptions.java | 47 + .../hadoop/test/TestGenericTestUtils.java | 44 + .../hadoop/util/TestAutoCloseableLock.java | 95 + .../hadoop/util/TestBasicDiskValidator.java | 51 + .../org/apache/hadoop/util/TestDiskChecker.java | 47 +- .../hadoop/util/TestDiskValidatorFactory.java | 59 + .../hadoop/util/TestGenericOptionsParser.java | 74 +- .../apache/hadoop/util/TestHostsFileReader.java | 64 +- .../org/apache/hadoop/util/TestLineReader.java | 59 +- .../java/org/apache/hadoop/util/TestShell.java | 18 +- .../org/apache/hadoop/util/TestStringUtils.java | 44 +- .../org/apache/hadoop/util/TestWinUtils.java | 4 +- .../hadoop/util/bloom/TestBloomFilters.java | 12 + .../src/test/resources/core-site.xml | 12 - .../src/test/resources/testConf.xml | 4 +- .../src/test/scripts/hadoop_slaves.bats | 37 - .../src/test/scripts/hadoop_ssh.bats | 18 +- .../src/test/scripts/hadoop_workers.bats | 37 + hadoop-common-project/hadoop-kms/pom.xml | 4 +- .../hadoop/crypto/key/kms/server/KMSACLs.java | 75 +- .../hadoop/crypto/key/kms/server/KMSAudit.java | 6 + .../hadoop-kms/src/site/markdown/index.md.vm | 68 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 9 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 394 +- .../crypto/key/kms/server/TestKMSACLs.java | 174 +- .../crypto/key/kms/server/TestKMSAudit.java | 53 +- .../src/test/resources/log4j.properties | 2 +- hadoop-common-project/hadoop-minikdc/pom.xml | 4 +- .../java/org/apache/hadoop/minikdc/MiniKdc.java | 16 +- hadoop-common-project/hadoop-nfs/pom.xml | 4 +- .../org/apache/hadoop/mount/MountInterface.java | 47 +- .../org/apache/hadoop/mount/MountResponse.java | 31 +- .../org/apache/hadoop/mount/MountdBase.java | 4 +- .../java/org/apache/hadoop/nfs/NfsExports.java | 1 + .../java/org/apache/hadoop/nfs/NfsTime.java | 1 + .../org/apache/hadoop/nfs/nfs3/FileHandle.java | 5 +- .../apache/hadoop/nfs/nfs3/Nfs3Constant.java | 6 +- .../apache/hadoop/nfs/nfs3/Nfs3Interface.java | 151 +- .../hadoop/nfs/nfs3/request/CREATE3Request.java | 6 +- .../hadoop/nfs/nfs3/request/LINK3Request.java | 5 +- .../hadoop/nfs/nfs3/request/LOOKUP3Request.java | 8 +- .../hadoop/nfs/nfs3/request/MKDIR3Request.java | 8 +- .../hadoop/nfs/nfs3/request/MKNOD3Request.java | 4 +- .../hadoop/nfs/nfs3/request/NFS3Request.java | 1 + .../hadoop/nfs/nfs3/request/REMOVE3Request.java | 8 +- .../hadoop/nfs/nfs3/request/RENAME3Request.java | 12 +- .../hadoop/nfs/nfs3/request/RMDIR3Request.java | 8 +- .../nfs/nfs3/request/SYMLINK3Request.java | 12 +- .../hadoop/nfs/nfs3/response/NFS3Response.java | 4 + .../org/apache/hadoop/oncrpc/RpcCallCache.java | 24 +- .../org/apache/hadoop/oncrpc/RpcProgram.java | 6 + .../main/java/org/apache/hadoop/oncrpc/XDR.java | 28 +- .../hadoop/oncrpc/security/Credentials.java | 8 + .../hadoop/oncrpc/security/CredentialsSys.java | 21 +- .../hadoop/oncrpc/security/RpcAuthInfo.java | 9 +- .../hadoop/oncrpc/security/SecurityHandler.java | 34 +- .../apache/hadoop/oncrpc/security/Verifier.java | 10 +- .../oncrpc/security/TestCredentialsSys.java | 42 + hadoop-common-project/pom.xml | 4 +- hadoop-dist/pom.xml | 4 +- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 35 +- .../hadoop/hdfs/AsyncDistributedFileSystem.java | 213 - .../java/org/apache/hadoop/hdfs/DFSClient.java | 16 +- .../hadoop/hdfs/DFSOpsCountStatistics.java | 132 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 74 +- .../hadoop/hdfs/DFSStripedInputStream.java | 18 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 42 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 33 +- .../hadoop/hdfs/DistributedFileSystem.java | 10 - .../apache/hadoop/hdfs/FileChecksumHelper.java | 13 +- .../hdfs/client/HdfsClientConfigKeys.java | 3 - .../hdfs/client/impl/BlockReaderFactory.java | 18 +- .../hdfs/client/impl/BlockReaderRemote.java | 570 +- .../hdfs/client/impl/BlockReaderRemote2.java | 474 - .../hadoop/hdfs/client/impl/DfsClientConf.java | 12 - .../hadoop/hdfs/protocol/CachePoolInfo.java | 25 + .../hdfs/protocol/ClientDatanodeProtocol.java | 31 + .../datatransfer/DataTransferProtocol.java | 10 +- .../hdfs/protocol/datatransfer/Sender.java | 4 +- .../ClientDatanodeProtocolTranslatorPB.java | 95 + .../hadoop/hdfs/protocolPB/PBHelperClient.java | 7 + .../delegation/DelegationTokenIdentifier.java | 26 + .../server/datanode/DiskBalancerWorkItem.java | 290 + .../server/datanode/DiskBalancerWorkStatus.java | 292 + .../hdfs/util/CombinedHostsFileReader.java | 11 +- .../hdfs/util/CombinedHostsFileWriter.java | 4 +- .../hdfs/web/resources/PermissionParam.java | 36 +- .../src/main/proto/ClientDatanodeProtocol.proto | 88 + .../src/main/proto/ClientNamenodeProtocol.proto | 1 + .../src/main/proto/datatransfer.proto | 1 + .../hadoop/hdfs/TestDFSOpsCountStatistics.java | 216 + hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +- .../src/main/conf/httpfs-env.sh | 4 + .../hadoop/fs/http/client/HttpFSUtils.java | 5 +- .../http/server/HttpFSAuthenticationFilter.java | 4 +- .../apache/hadoop/lib/wsrs/JSONMapProvider.java | 5 +- .../apache/hadoop/lib/wsrs/JSONProvider.java | 5 +- .../src/main/libexec/httpfs-config.sh | 1 + .../src/main/tomcat/server.xml | 1 + .../src/main/tomcat/ssl-server.xml.conf | 1 + .../src/site/markdown/ServerSetup.md.vm | 4 + .../org/apache/hadoop/test/TestJettyHelper.java | 6 +- .../hadoop-hdfs-native-client/pom.xml | 9 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 +- .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 4 +- .../bkjournal/BookKeeperJournalManager.java | 10 +- .../hadoop-hdfs/src/main/bin/hdfs | 16 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 12 +- .../src/main/bin/start-secure-dns.sh | 2 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 12 +- .../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 2 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 48 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 191 +- .../apache/hadoop/hdfs/HDFSPolicyProvider.java | 6 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 35 + .../hadoop/hdfs/protocol/BlockListAsLongs.java | 4 +- .../hadoop/hdfs/protocol/FSLimitException.java | 4 +- .../hdfs/protocol/datatransfer/Receiver.java | 3 +- ...tDatanodeProtocolServerSideTranslatorPB.java | 93 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 6 +- .../qjournal/client/QuorumJournalManager.java | 28 +- .../qjournal/client/QuorumOutputStream.java | 13 +- .../hadoop/hdfs/qjournal/server/Journal.java | 20 +- .../token/block/BlockTokenSecretManager.java | 47 +- .../hadoop/hdfs/server/balancer/Balancer.java | 2 + .../hadoop/hdfs/server/balancer/Dispatcher.java | 17 +- .../AvailableSpaceBlockPlacementPolicy.java | 16 +- .../server/blockmanagement/BlockManager.java | 102 +- .../blockmanagement/InvalidateBlocks.java | 3 +- .../PendingReconstructionBlocks.java | 4 +- .../hdfs/server/datanode/BPServiceActor.java | 106 +- .../server/datanode/BlockChecksumHelper.java | 81 +- .../server/datanode/BlockPoolSliceStorage.java | 2 +- .../hdfs/server/datanode/BlockReceiver.java | 8 +- .../hdfs/server/datanode/BlockSender.java | 3 +- .../hadoop/hdfs/server/datanode/DNConf.java | 15 + .../hadoop/hdfs/server/datanode/DataNode.java | 209 +- .../server/datanode/DataNodeFaultInjector.java | 2 + .../hdfs/server/datanode/DataNodeMXBean.java | 28 +- .../hdfs/server/datanode/DataXceiver.java | 10 +- .../hdfs/server/datanode/DataXceiverServer.java | 4 + .../hdfs/server/datanode/DirectoryScanner.java | 3 +- .../hdfs/server/datanode/DiskBalancer.java | 1067 ++ .../hdfs/server/datanode/VolumeScanner.java | 2 +- .../StripedBlockChecksumReconstructor.java | 72 +- .../erasurecode/StripedBlockReader.java | 8 +- .../erasurecode/StripedBlockReconstructor.java | 4 +- .../erasurecode/StripedBlockWriter.java | 24 +- .../datanode/erasurecode/StripedReader.java | 11 +- .../erasurecode/StripedReconstructor.java | 13 +- .../datanode/erasurecode/StripedWriter.java | 8 + .../server/datanode/fsdataset/FsDatasetSpi.java | 17 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 1025 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 22 +- .../datanode/metrics/DataNodeMetrics.java | 16 + .../server/datanode/web/DatanodeHttpServer.java | 10 + .../web/webhdfs/DataNodeUGIProvider.java | 7 + .../datanode/web/webhdfs/ParameterParser.java | 12 +- .../datanode/web/webhdfs/WebHdfsHandler.java | 49 +- .../diskbalancer/DiskBalancerConstants.java | 44 + .../diskbalancer/DiskBalancerException.java | 101 + .../diskbalancer/command/CancelCommand.java | 147 + .../server/diskbalancer/command/Command.java | 474 + .../diskbalancer/command/ExecuteCommand.java | 116 + .../diskbalancer/command/HelpCommand.java | 109 + .../diskbalancer/command/PlanCommand.java | 237 + .../diskbalancer/command/QueryCommand.java | 106 + .../diskbalancer/command/ReportCommand.java | 216 + .../diskbalancer/command/package-info.java | 22 + .../connectors/ClusterConnector.java | 44 + .../connectors/ConnectorFactory.java | 55 + .../connectors/DBNameNodeConnector.java | 162 + .../connectors/JsonNodeConnector.java | 80 + .../diskbalancer/connectors/package-info.java | 37 + .../datamodel/DiskBalancerCluster.java | 394 + .../datamodel/DiskBalancerDataNode.java | 270 + .../datamodel/DiskBalancerVolume.java | 364 + .../datamodel/DiskBalancerVolumeSet.java | 342 + .../diskbalancer/datamodel/package-info.java | 31 + .../hdfs/server/diskbalancer/package-info.java | 36 + .../diskbalancer/planner/GreedyPlanner.java | 269 + .../server/diskbalancer/planner/MoveStep.java | 262 + .../server/diskbalancer/planner/NodePlan.java | 190 + .../server/diskbalancer/planner/Planner.java | 28 + .../diskbalancer/planner/PlannerFactory.java | 60 + .../hdfs/server/diskbalancer/planner/Step.java | 109 + .../diskbalancer/planner/package-info.java | 46 + .../apache/hadoop/hdfs/server/mover/Mover.java | 15 +- .../server/namenode/BackupJournalManager.java | 2 +- .../hdfs/server/namenode/CacheManager.java | 19 +- .../hadoop/hdfs/server/namenode/CachePool.java | 28 +- .../namenode/ErasureCodingPolicyManager.java | 20 + .../hadoop/hdfs/server/namenode/FSDirAclOp.java | 35 +- .../hdfs/server/namenode/FSDirAppendOp.java | 6 +- .../hdfs/server/namenode/FSDirAttrOp.java | 35 +- .../hdfs/server/namenode/FSDirDeleteOp.java | 7 +- .../server/namenode/FSDirEncryptionZoneOp.java | 41 +- .../server/namenode/FSDirErasureCodingOp.java | 12 +- .../hdfs/server/namenode/FSDirMkdirOp.java | 10 +- .../hdfs/server/namenode/FSDirRenameOp.java | 45 +- .../server/namenode/FSDirStatAndListingOp.java | 69 +- .../hdfs/server/namenode/FSDirSymlinkOp.java | 5 +- .../hdfs/server/namenode/FSDirTruncateOp.java | 6 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 93 +- .../hdfs/server/namenode/FSDirXAttrOp.java | 34 +- .../hdfs/server/namenode/FSDirectory.java | 258 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 28 +- .../server/namenode/FSImageFormatPBINode.java | 10 +- .../server/namenode/FSImageFormatProtobuf.java | 6 +- .../server/namenode/FSImageSerialization.java | 25 +- .../hdfs/server/namenode/FSImageUtil.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 60 +- .../server/namenode/FSPermissionChecker.java | 196 +- .../server/namenode/FileJournalManager.java | 9 +- .../hadoop/hdfs/server/namenode/INode.java | 54 +- .../server/namenode/INodeAttributeProvider.java | 10 + .../hadoop/hdfs/server/namenode/INodeFile.java | 19 +- .../hdfs/server/namenode/INodesInPath.java | 12 +- .../hadoop/hdfs/server/namenode/JournalSet.java | 11 +- .../hdfs/server/namenode/LogsPurgeable.java | 6 +- .../hadoop/hdfs/server/namenode/NNStorage.java | 161 +- .../namenode/NNStorageRetentionManager.java | 2 +- .../server/namenode/NameNodeHttpServer.java | 20 + .../hdfs/server/namenode/SecondaryNameNode.java | 2 +- .../hdfs/server/namenode/ha/EditLogTailer.java | 92 +- .../hadoop/hdfs/server/namenode/ha/HAState.java | 6 +- .../namenode/metrics/NameNodeMetrics.java | 26 + .../web/resources/NamenodeWebHdfsMethods.java | 8 +- .../server/protocol/RemoteEditLogManifest.java | 19 +- .../apache/hadoop/hdfs/tools/CacheAdmin.java | 34 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 34 +- .../apache/hadoop/hdfs/tools/DebugAdmin.java | 4 +- .../apache/hadoop/hdfs/tools/DiskBalancer.java | 482 + .../FileDistributionCalculator.java | 6 + .../FileDistributionVisitor.java | 4 + .../hadoop-hdfs/src/main/java/overview.html | 46 +- .../hadoop-hdfs/src/main/proto/HdfsServer.proto | 1 + .../src/main/resources/hdfs-default.xml | 107 +- .../src/main/webapps/datanode/datanode.html | 131 + .../hadoop-hdfs/src/main/webapps/datanode/dn.js | 70 + .../src/main/webapps/datanode/index.html | 48 +- .../src/main/webapps/hdfs/dfshealth.html | 12 + .../src/main/webapps/hdfs/dfshealth.js | 11 + .../src/site/markdown/ArchivalStorage.md | 4 +- .../hadoop-hdfs/src/site/markdown/Federation.md | 2 +- .../src/site/markdown/HDFSCommands.md | 28 +- .../src/site/markdown/HDFSDiskbalancer.md | 120 + .../markdown/HDFSHighAvailabilityWithQJM.md | 2 +- .../src/site/markdown/HdfsQuotaAdminGuide.md | 7 +- .../src/site/markdown/HdfsRollingUpgrade.md | 2 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 6 +- .../org/apache/hadoop/fs/TestGlobPaths.java | 4 +- .../hadoop/fs/permission/TestStickyBit.java | 58 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 7 +- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 2 + .../org/apache/hadoop/hdfs/TestAsyncDFS.java | 454 - .../apache/hadoop/hdfs/TestAsyncDFSRename.java | 187 - .../apache/hadoop/hdfs/TestAsyncHDFSWithHA.java | 181 - .../hadoop/hdfs/TestDFSClientSocketSize.java | 80 +- .../apache/hadoop/hdfs/TestDFSInputStream.java | 1 - .../apache/hadoop/hdfs/TestDFSPermission.java | 9 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 91 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 8 + .../hadoop/hdfs/TestDFSStripedOutputStream.java | 8 + .../TestDFSStripedOutputStreamWithFailure.java | 11 +- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 98 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 74 +- .../apache/hadoop/hdfs/TestFileChecksum.java | 309 +- .../hadoop/hdfs/TestHDFSPolicyProvider.java | 31 +- .../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 82 +- .../apache/hadoop/hdfs/TestMiniDFSCluster.java | 53 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 11 +- .../hdfs/TestSecureEncryptionZoneWithKMS.java | 305 + .../hdfs/client/impl/TestBlockReaderBase.java | 97 - .../hdfs/client/impl/TestBlockReaderRemote.java | 80 +- .../client/impl/TestBlockReaderRemote2.java | 27 - .../impl/TestClientBlockVerification.java | 10 +- .../hdfs/protocol/TestBlockListAsLongs.java | 19 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 2 +- .../hdfs/qjournal/server/TestJournal.java | 4 +- .../hdfs/security/TestDelegationToken.java | 31 +- .../hdfs/server/balancer/TestBalancer.java | 8 +- .../TestAvailableSpaceBlockPlacementPolicy.java | 18 + .../blockmanagement/TestBlockManager.java | 105 +- .../TestRBWBlockInvalidation.java | 5 +- .../hdfs/server/common/TestJspHelper.java | 8 +- .../server/datanode/SimulatedFSDataset.java | 16 + .../hdfs/server/datanode/TestBlockRecovery.java | 3 +- .../TestDataNodeErasureCodingMetrics.java | 46 +- .../datanode/TestDataNodeHotSwapVolumes.java | 27 +- .../server/datanode/TestDataNodeMXBean.java | 76 +- .../server/datanode/TestDataNodeMetrics.java | 31 + .../server/datanode/TestDataNodeTcpNoDelay.java | 460 + .../datanode/TestDataNodeVolumeFailure.java | 6 +- .../TestDataNodeVolumeFailureReporting.java | 24 +- .../TestDataNodeVolumeFailureToleration.java | 10 +- .../server/datanode/TestDirectoryScanner.java | 9 +- .../hdfs/server/datanode/TestDiskError.java | 55 + .../datanode/TestFsDatasetCacheRevocation.java | 7 +- .../extdataset/ExternalDatasetImpl.java | 13 + .../fsdataset/impl/LazyPersistTestCase.java | 30 +- .../fsdataset/impl/TestFsDatasetImpl.java | 43 + .../fsdataset/impl/TestLazyPersistFiles.java | 9 +- .../impl/TestLazyPersistLockedMemory.java | 3 +- .../impl/TestLazyPersistReplicaPlacement.java | 10 +- .../impl/TestLazyPersistReplicaRecovery.java | 5 +- .../fsdataset/impl/TestScrLazyPersistFiles.java | 5 +- .../fsdataset/impl/TestSpaceReservation.java | 48 + .../datanode/web/TestDatanodeHttpXFrame.java | 90 + .../web/webhdfs/TestDataNodeUGIProvider.java | 1 + .../DiskBalancerResultVerifier.java | 45 + .../diskbalancer/DiskBalancerTestUtil.java | 267 + .../server/diskbalancer/TestConnectors.java | 85 + .../server/diskbalancer/TestDataModels.java | 227 + .../server/diskbalancer/TestDiskBalancer.java | 356 + .../diskbalancer/TestDiskBalancerRPC.java | 324 + .../TestDiskBalancerWithMockMover.java | 629 ++ .../hdfs/server/diskbalancer/TestPlanner.java | 513 + .../command/TestDiskBalancerCommand.java | 438 + .../diskbalancer/connectors/NullConnector.java | 59 + .../hadoop/hdfs/server/mover/TestMover.java | 32 + .../namenode/MockNameNodeResourceChecker.java | 49 + .../server/namenode/NNThroughputBenchmark.java | 32 +- .../hdfs/server/namenode/TestCheckpoint.java | 7 +- .../hdfs/server/namenode/TestEditLog.java | 27 +- .../server/namenode/TestFileJournalManager.java | 2 +- .../hdfs/server/namenode/TestFsLimits.java | 2 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 4 +- .../server/namenode/TestGenericJournalConf.java | 2 +- .../server/namenode/TestGetBlockLocations.java | 4 +- .../hdfs/server/namenode/TestINodeFile.java | 53 +- .../hdfs/server/namenode/TestLeaseManager.java | 47 + .../hdfs/server/namenode/TestMetaSave.java | 1 + .../namenode/TestNNStorageRetentionManager.java | 4 +- .../namenode/TestNameNodeHttpServerXFrame.java | 97 + .../namenode/TestNameNodeResourceChecker.java | 11 +- .../server/namenode/TestPathComponents.java | 100 +- .../namenode/TestReconstructStripedBlocks.java | 2 + .../hdfs/server/namenode/TestSaveNamespace.java | 116 +- .../server/namenode/TestSnapshotPathINodes.java | 37 +- .../server/namenode/TestStripedINodeFile.java | 87 +- .../namenode/ha/TestDelegationTokensWithHA.java | 32 + .../server/namenode/ha/TestEditLogTailer.java | 92 + .../namenode/ha/TestFailureToReadEdits.java | 7 +- .../server/namenode/ha/TestNNHealthCheck.java | 13 +- .../namenode/ha/TestStandbyInProgressTail.java | 339 + .../namenode/metrics/TestNameNodeMetrics.java | 59 + .../snapshot/TestOpenFilesWithSnapshot.java | 21 +- .../resources/TestWebHdfsCreatePermissions.java | 124 + .../shortcircuit/TestShortCircuitLocalRead.java | 16 +- .../hdfs/tools/TestDFSZKFailoverController.java | 11 +- .../TestOfflineImageViewer.java | 49 + .../hdfs/util/TestAtomicFileOutputStream.java | 5 +- .../hadoop/hdfs/web/resources/TestParam.java | 3 +- .../apache/hadoop/security/TestPermission.java | 12 + .../security/TestRefreshUserMappings.java | 20 +- .../TestTracingShortCircuitLocalRead.java | 5 +- .../diskBalancer/data-cluster-3node-3disk.json | 380 + .../diskBalancer/data-cluster-64node-3disk.json | 9484 ++++++++++++++++++ .../src/test/resources/testCacheAdminConf.xml | 51 +- .../src/test/resources/testHDFSConf.xml | 20 + hadoop-hdfs-project/pom.xml | 4 +- .../hadoop-mapreduce-client-app/pom.xml | 4 +- .../jobhistory/JobHistoryEventHandler.java | 342 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 18 + .../mapreduce/v2/app/job/impl/JobImpl.java | 2 +- .../v2/app/job/impl/TaskAttemptImpl.java | 3 +- .../v2/app/rm/RMContainerAllocator.java | 14 +- .../v2/app/webapp/SingleCounterBlock.java | 14 +- .../mapreduce/v2/app/webapp/dao/ConfInfo.java | 4 +- .../hadoop/mapreduce/jobhistory/TestEvents.java | 18 +- .../jobhistory/TestJobHistoryEventHandler.java | 109 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 2 +- .../v2/app/job/impl/TestTaskAttempt.java | 5 +- .../v2/app/rm/TestRMContainerAllocator.java | 3 +- .../v2/app/webapp/TestAMWebServices.java | 33 +- .../v2/app/webapp/TestAMWebServicesAttempt.java | 24 +- .../app/webapp/TestAMWebServicesAttempts.java | 31 +- .../v2/app/webapp/TestAMWebServicesJobConf.java | 31 +- .../v2/app/webapp/TestAMWebServicesJobs.java | 35 +- .../v2/app/webapp/TestAMWebServicesTasks.java | 36 +- .../mapreduce/v2/app/webapp/TestBlocks.java | 129 +- .../hadoop-mapreduce-client-common/pom.xml | 4 +- .../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 + .../apache/hadoop/mapreduce/v2/util/MRApps.java | 70 +- .../hadoop/mapreduce/TestTypeConverter.java | 4 +- .../hadoop-mapreduce-client-core/pom.xml | 4 +- .../hadoop/filecache/DistributedCache.java | 70 +- .../apache/hadoop/mapred/FileInputFormat.java | 7 + .../apache/hadoop/mapred/TextOutputFormat.java | 24 +- .../org/apache/hadoop/mapreduce/Cluster.java | 4 + .../java/org/apache/hadoop/mapreduce/Job.java | 7 +- .../hadoop/mapreduce/JobResourceUploader.java | 258 +- .../hadoop/mapreduce/JobSubmissionFiles.java | 4 +- .../apache/hadoop/mapreduce/JobSubmitter.java | 8 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 33 + .../ClientDistributedCacheManager.java | 46 +- .../mapreduce/filecache/DistributedCache.java | 184 +- .../mapreduce/jobhistory/AMStartedEvent.java | 28 +- .../mapreduce/jobhistory/HistoryEvent.java | 18 + .../HumanReadableHistoryViewerPrinter.java | 15 +- .../mapreduce/jobhistory/JobFinishedEvent.java | 33 + .../jobhistory/JobInfoChangeEvent.java | 21 +- .../mapreduce/jobhistory/JobInitedEvent.java | 25 +- .../jobhistory/JobPriorityChangeEvent.java | 21 +- .../jobhistory/JobQueueChangeEvent.java | 18 + .../jobhistory/JobStatusChangedEvent.java | 21 +- .../mapreduce/jobhistory/JobSubmittedEvent.java | 67 +- .../JobUnsuccessfulCompletionEvent.java | 27 +- .../jobhistory/MapAttemptFinishedEvent.java | 32 +- .../jobhistory/NormalizedResourceEvent.java | 19 + .../jobhistory/ReduceAttemptFinishedEvent.java | 33 +- .../jobhistory/TaskAttemptFinishedEvent.java | 26 + .../jobhistory/TaskAttemptStartedEvent.java | 29 +- .../TaskAttemptUnsuccessfulCompletionEvent.java | 37 +- .../mapreduce/jobhistory/TaskFailedEvent.java | 29 +- .../mapreduce/jobhistory/TaskFinishedEvent.java | 27 +- .../mapreduce/jobhistory/TaskStartedEvent.java | 20 + .../mapreduce/jobhistory/TaskUpdatedEvent.java | 21 +- .../lib/input/CombineFileInputFormat.java | 4 + .../mapreduce/lib/input/FileInputFormat.java | 7 + .../mapreduce/lib/output/TextOutputFormat.java | 24 +- .../mapreduce/task/reduce/MergeManagerImpl.java | 28 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 4 +- .../mapreduce/util/JobHistoryEventUtils.java | 83 + .../hadoop/mapreduce/util/MRJobConfUtil.java | 45 + .../src/main/resources/mapred-default.xml | 68 +- .../src/site/markdown/MapReduceTutorial.md | 24 +- .../hadoop/mapred/TestLineRecordReader.java | 10 +- .../mapreduce/TestJobResourceUploader.java | 355 + .../TestClientDistributedCacheManager.java | 151 +- .../filecache/TestDistributedCache.java | 132 + .../lib/input/TestLineRecordReader.java | 8 +- .../mapreduce/task/reduce/TestMergeManager.java | 25 + .../apache/hadoop/mapreduce/tools/TestCLI.java | 47 +- .../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +- .../hadoop-mapreduce-client-hs/pom.xml | 4 +- .../mapreduce/v2/hs/HistoryFileManager.java | 35 +- .../hadoop/mapreduce/v2/hs/UnparsedJob.java | 211 + .../mapreduce/v2/hs/webapp/HsJobBlock.java | 16 +- .../mapreduce/v2/hs/TestHistoryFileManager.java | 98 +- .../mapreduce/v2/hs/TestJobListCache.java | 2 +- .../v2/hs/server/TestHSAdminServer.java | 3 +- .../mapreduce/v2/hs/webapp/TestHsJobBlock.java | 268 + .../v2/hs/webapp/TestHsWebServices.java | 32 +- .../v2/hs/webapp/TestHsWebServicesAttempts.java | 38 +- .../v2/hs/webapp/TestHsWebServicesJobConf.java | 24 +- .../v2/hs/webapp/TestHsWebServicesJobs.java | 40 +- .../hs/webapp/TestHsWebServicesJobsQuery.java | 64 +- .../v2/hs/webapp/TestHsWebServicesTasks.java | 36 +- .../resources/job_1329348432655_0001_conf.xml | 13 +- .../hadoop-mapreduce-client-jobclient/pom.xml | 10 +- .../org/apache/hadoop/mapred/YARNRunner.java | 2 +- .../java/org/apache/hadoop/fs/TestDFSIO.java | 64 +- .../apache/hadoop/mapred/ReliabilityTest.java | 119 +- .../apache/hadoop/mapred/TestLazyOutput.java | 31 +- .../hadoop/mapred/TestLocalJobSubmission.java | 34 +- .../mapred/TestMRCJCFileOutputCommitter.java | 14 +- .../hadoop/mapred/TestMROpportunisticMaps.java | 2 + .../mapred/TestMRTimelineEventHandling.java | 357 +- .../apache/hadoop/mapred/TestYARNRunner.java | 6 +- .../org/apache/hadoop/mapred/UtilsForTests.java | 47 +- .../apache/hadoop/mapred/pipes/TestPipes.java | 105 +- .../apache/hadoop/mapreduce/EntityWriterV2.java | 57 + .../hadoop/mapreduce/JobHistoryFileParser.java | 3 + .../mapreduce/JobHistoryFileReplayMapperV1.java | 14 +- .../mapreduce/JobHistoryFileReplayMapperV2.java | 161 + .../mapreduce/SimpleEntityWriterConstants.java | 43 + .../hadoop/mapreduce/SimpleEntityWriterV1.java | 32 +- .../hadoop/mapreduce/SimpleEntityWriterV2.java | 131 + .../mapreduce/TestMapReduceLazyOutput.java | 44 +- .../mapreduce/TimelineEntityConverterV1.java | 15 +- .../mapreduce/TimelineEntityConverterV2.java | 206 + .../mapreduce/TimelineServicePerformance.java | 130 +- .../mapreduce/security/TestBinaryTokenFile.java | 68 +- .../mapreduce/security/TestMRCredentials.java | 6 +- .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 22 +- .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 214 +- .../apache/hadoop/test/MapredTestDriver.java | 32 +- .../hadoop-mapreduce-client-nativetask/pom.xml | 4 +- .../hadoop-mapreduce-client-shuffle/pom.xml | 4 +- .../apache/hadoop/mapred/ShuffleHandler.java | 223 +- .../hadoop/mapred/TestShuffleHandler.java | 16 +- .../hadoop-mapreduce-client/pom.xml | 4 +- .../hadoop-mapreduce-examples/pom.xml | 4 +- hadoop-mapreduce-project/pom.xml | 4 +- hadoop-maven-plugins/pom.xml | 2 +- hadoop-minicluster/pom.xml | 4 +- hadoop-project-dist/pom.xml | 4 +- hadoop-project/pom.xml | 218 +- hadoop-project/src/site/markdown/index.md.vm | 200 +- hadoop-project/src/site/site.xml | 7 +- hadoop-tools/hadoop-archive-logs/pom.xml | 4 +- hadoop-tools/hadoop-archives/pom.xml | 4 +- hadoop-tools/hadoop-aws/pom.xml | 4 +- .../java/org/apache/hadoop/fs/s3/Block.java | 52 - .../apache/hadoop/fs/s3/FileSystemStore.java | 67 - .../java/org/apache/hadoop/fs/s3/INode.java | 128 - .../hadoop/fs/s3/Jets3tFileSystemStore.java | 449 - .../org/apache/hadoop/fs/s3/MigrationTool.java | 291 - .../org/apache/hadoop/fs/s3/S3Credentials.java | 102 - .../org/apache/hadoop/fs/s3/S3Exception.java | 39 - .../org/apache/hadoop/fs/s3/S3FileSystem.java | 516 - .../hadoop/fs/s3/S3FileSystemConfigKeys.java | 47 - .../hadoop/fs/s3/S3FileSystemException.java | 36 - .../org/apache/hadoop/fs/s3/S3InputStream.java | 220 - .../org/apache/hadoop/fs/s3/S3OutputStream.java | 235 - .../hadoop/fs/s3/VersionMismatchException.java | 37 - .../java/org/apache/hadoop/fs/s3/package.html | 55 - .../org/apache/hadoop/fs/s3a/Constants.java | 48 +- .../java/org/apache/hadoop/fs/s3a/Listing.java | 594 ++ .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 457 +- .../apache/hadoop/fs/s3a/S3AInputPolicy.java | 76 + .../apache/hadoop/fs/s3a/S3AInputStream.java | 187 +- .../hadoop/fs/s3a/S3AInstrumentation.java | 21 +- .../hadoop/fs/s3a/S3AStorageStatistics.java | 17 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 12 + .../org/apache/hadoop/fs/s3a/Statistic.java | 78 +- .../s3native/Jets3tNativeFileSystemStore.java | 2 - .../hadoop/fs/s3native/NativeS3FileSystem.java | 34 +- .../hadoop/fs/s3native/S3Credentials.java | 100 + .../apache/hadoop/fs/s3native/S3Exception.java | 39 + .../s3native/S3NativeFileSystemConfigKeys.java | 21 +- .../org/apache/hadoop/fs/s3native/package.html | 5 +- .../services/org.apache.hadoop.fs.FileSystem | 1 - .../src/site/markdown/tools/hadoop-aws/index.md | 681 +- .../hadoop/fs/contract/s3/S3Contract.java | 50 - .../fs/contract/s3/TestS3ContractCreate.java | 32 - .../fs/contract/s3/TestS3ContractDelete.java | 31 - .../fs/contract/s3/TestS3ContractMkdir.java | 32 - .../fs/contract/s3/TestS3ContractOpen.java | 32 - .../fs/contract/s3/TestS3ContractRename.java | 32 - .../fs/contract/s3/TestS3ContractRootDir.java | 49 - .../fs/contract/s3/TestS3ContractSeek.java | 41 - .../fs/contract/s3a/TestS3AContractCreate.java | 5 - .../s3a/TestS3AContractGetFileStatus.java | 16 + .../hadoop/fs/s3/InMemoryFileSystemStore.java | 200 - .../fs/s3/Jets3tS3FileSystemContractTest.java | 31 - .../fs/s3/S3FileSystemContractBaseTest.java | 61 - .../hadoop/fs/s3/S3InMemoryFileSystem.java | 32 - .../java/org/apache/hadoop/fs/s3/TestINode.java | 60 - .../fs/s3/TestInMemoryS3FileSystemContract.java | 31 - .../apache/hadoop/fs/s3/TestS3Credentials.java | 140 - .../apache/hadoop/fs/s3/TestS3FileSystem.java | 52 - .../hadoop/fs/s3/TestS3InMemoryFileSystem.java | 67 - .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 23 +- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 1 + .../hadoop/fs/s3a/TestS3ACredentialsInURL.java | 16 +- .../hadoop/fs/s3a/TestS3AInputPolicies.java | 91 + .../fs/s3a/TestS3ATemporaryCredentials.java | 24 +- .../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 15 +- .../s3a/scale/TestS3ADirectoryPerformance.java | 58 +- .../scale/TestS3AInputStreamPerformance.java | 403 +- .../s3native/InMemoryNativeFileSystemStore.java | 11 +- .../hadoop/fs/s3native/TestS3Credentials.java | 147 + .../src/test/resources/contract/s3.xml | 104 - hadoop-tools/hadoop-azure-datalake/pom.xml | 9 +- .../web/PrivateAzureDataLakeFileSystem.java | 8 + ...ClientCredentialBasedAccesTokenProvider.java | 155 + .../src/site/markdown/index.md | 62 +- .../hdfs/web/TestConfigurationSetting.java | 54 +- hadoop-tools/hadoop-azure/pom.xml | 2 +- .../AzureFileSystemThreadPoolExecutor.java | 346 + .../fs/azure/AzureFileSystemThreadTask.java | 29 + .../fs/azure/AzureNativeFileSystemStore.java | 53 +- .../hadoop/fs/azure/BlockBlobAppendStream.java | 99 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 235 +- .../hadoop/fs/azure/SendRequestIntercept.java | 2 +- .../hadoop/fs/azure/StorageInterfaceImpl.java | 2 +- .../hadoop-azure/src/site/markdown/index.md | 19 + .../hadoop/fs/azure/AbstractWasbTestBase.java | 66 + .../fs/azure/NativeAzureFileSystemBaseTest.java | 28 +- ...estFileSystemOperationExceptionHandling.java | 4 +- ...perationsExceptionHandlingMultiThreaded.java | 6 +- .../TestFileSystemOperationsWithThreads.java | 720 ++ .../azure/TestNativeAzureFileSystemAppend.java | 2 +- ...ativeAzureFileSystemAtomicRenameDirList.java | 3 +- .../TestNativeAzureFileSystemClientLogging.java | 6 +- ...stNativeAzureFileSystemOperationsMocked.java | 4 +- .../azure/TestShellDecryptionKeyProvider.java | 10 +- .../src/test/resources/log4j.properties | 4 +- hadoop-tools/hadoop-datajoin/pom.xml | 4 +- hadoop-tools/hadoop-distcp/pom.xml | 4 +- .../java/org/apache/hadoop/tools/DiffInfo.java | 4 +- .../org/apache/hadoop/tools/DistCpOptions.java | 4 + .../apache/hadoop/tools/TestDistCpSystem.java | 199 +- .../apache/hadoop/tools/TestOptionsParser.java | 1 + hadoop-tools/hadoop-extras/pom.xml | 4 +- hadoop-tools/hadoop-gridmix/pom.xml | 4 +- hadoop-tools/hadoop-kafka/pom.xml | 4 +- .../apache/hadoop/metrics2/sink/KafkaSink.java | 23 +- hadoop-tools/hadoop-openstack/pom.xml | 5 +- hadoop-tools/hadoop-pipes/pom.xml | 4 +- hadoop-tools/hadoop-rumen/pom.xml | 4 +- hadoop-tools/hadoop-sls/pom.xml | 4 +- .../src/main/data/2jobs2min-rumen-jh.json | 36 +- .../org/apache/hadoop/yarn/sls/SLSRunner.java | 2 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 5 + .../yarn/sls/scheduler/RMNodeWrapper.java | 5 + .../sls/scheduler/ResourceSchedulerWrapper.java | 12 +- .../sls/scheduler/SLSCapacityScheduler.java | 12 +- hadoop-tools/hadoop-streaming/pom.xml | 4 +- hadoop-tools/hadoop-tools-dist/pom.xml | 4 +- hadoop-tools/pom.xml | 4 +- .../hadoop-yarn/bin/start-yarn.sh | 6 +- .../hadoop-yarn/bin/stop-yarn.sh | 6 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 19 +- .../hadoop-yarn/bin/yarn-config.cmd | 2 +- .../hadoop-yarn/bin/yarn-config.sh | 24 +- .../hadoop-yarn/bin/yarn-daemons.sh | 6 +- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 9 +- hadoop-yarn-project/hadoop-yarn/conf/slaves | 1 - .../hadoop-yarn/conf/yarn-env.sh | 14 +- .../dev-support/findbugs-exclude.xml | 39 + .../hadoop-yarn/hadoop-yarn-api/pom.xml | 8 +- .../hadoop/yarn/api/ApplicationConstants.java | 8 + .../api/protocolrecords/AllocateRequest.java | 12 +- .../api/protocolrecords/AllocateResponse.java | 36 + .../FinishApplicationMasterResponse.java | 4 + .../protocolrecords/GetApplicationsRequest.java | 4 + .../protocolrecords/GetClusterNodesRequest.java | 2 + .../GetContainerStatusesResponse.java | 1 + .../GetDelegationTokenResponse.java | 2 +- .../IncreaseContainersResourceResponse.java | 1 + .../KillApplicationResponse.java | 2 + .../RegisterApplicationMasterResponse.java | 2 + .../protocolrecords/ReservationListRequest.java | 3 + .../StartContainersResponse.java | 1 + .../protocolrecords/StopContainersResponse.java | 1 + .../yarn/api/records/DecommissionType.java | 12 +- .../yarn/api/records/ExecutionTypeRequest.java | 9 +- .../hadoop/yarn/api/records/LocalResource.java | 3 + .../hadoop/yarn/api/records/Resource.java | 47 +- .../yarn/api/records/ResourceRequest.java | 23 +- .../api/records/timeline/TimelineEntity.java | 21 +- .../api/records/timeline/TimelineEvent.java | 8 +- .../ApplicationAttemptEntity.java | 41 + .../timelineservice/ApplicationEntity.java | 52 + .../records/timelineservice/ClusterEntity.java | 40 + .../timelineservice/ContainerEntity.java | 41 + .../timelineservice/FlowActivityEntity.java | 191 + .../records/timelineservice/FlowRunEntity.java | 126 + .../HierarchicalTimelineEntity.java | 133 + .../records/timelineservice/QueueEntity.java | 40 + .../timelineservice/TimelineEntities.java | 62 + .../records/timelineservice/TimelineEntity.java | 584 ++ .../timelineservice/TimelineEntityType.java | 101 + .../records/timelineservice/TimelineEvent.java | 133 + .../records/timelineservice/TimelineMetric.java | 289 + .../TimelineMetricCalculator.java | 115 + .../TimelineMetricOperation.java | 167 + .../timelineservice/TimelineWriteResponse.java | 167 + .../api/records/timelineservice/UserEntity.java | 40 + .../records/timelineservice/package-info.java | 26 + .../org/apache/hadoop/yarn/conf/HAUtil.java | 5 +- .../hadoop/yarn/conf/YarnConfiguration.java | 301 +- .../protocolrecords/RefreshNodesRequest.java | 26 +- .../hadoop/yarn/util/TimelineServiceHelper.java | 49 + ..._server_resourcemanager_service_protos.proto | 1 + .../src/main/proto/yarn_protos.proto | 4 +- .../src/main/proto/yarn_service_protos.proto | 1 + .../timelineservice/TestTimelineMetric.java | 100 + .../yarn/conf/TestYarnConfigurationFields.java | 38 +- .../pom.xml | 14 +- .../distributedshell/ApplicationMaster.java | 178 +- .../applications/distributedshell/Client.java | 49 +- .../distributedshell/TestDistributedShell.java | 312 +- .../TestDistributedShellWithNodeLabels.java | 2 +- .../pom.xml | 4 +- .../hadoop-yarn-applications/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +- .../hadoop/yarn/client/api/AMRMClient.java | 20 +- .../yarn/client/api/async/AMRMClientAsync.java | 19 +- .../api/async/impl/AMRMClientAsyncImpl.java | 17 +- .../yarn/client/api/impl/YarnClientImpl.java | 19 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 828 +- .../apache/hadoop/yarn/client/cli/NodeCLI.java | 32 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 174 +- .../hadoop/yarn/client/ProtocolHATestBase.java | 4 +- .../TestApplicationClientProtocolOnHA.java | 2 +- .../yarn/client/api/impl/TestAMRMClient.java | 131 +- .../api/impl/TestDistributedScheduling.java | 131 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 511 +- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 94 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 31 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 16 +- .../impl/pb/AllocateResponsePBImpl.java | 17 + .../impl/pb/ExecutionTypeRequestPBImpl.java | 5 + .../yarn/api/records/impl/pb/ProtoUtils.java | 14 + .../api/records/impl/pb/ResourcePBImpl.java | 15 +- .../hadoop/yarn/client/api/TimelineClient.java | 87 +- .../api/impl/FileSystemTimelineWriter.java | 3 +- .../client/api/impl/TimelineClientImpl.java | 509 +- .../yarn/client/api/impl/TimelineWriter.java | 3 +- .../hadoop/yarn/event/AsyncDispatcher.java | 8 +- .../factory/providers/RpcFactoryProvider.java | 13 +- .../logaggregation/AggregatedLogFormat.java | 39 +- .../logaggregation/ContainerLogsRequest.java | 22 +- .../yarn/logaggregation/LogCLIHelpers.java | 128 +- .../hadoop/yarn/security/AccessRequest.java | 27 +- .../impl/pb/RefreshNodesRequestPBImpl.java | 17 +- .../resource/DefaultResourceCalculator.java | 2 +- .../resource/DominantResourceCalculator.java | 8 +- .../yarn/util/resource/ResourceCalculator.java | 17 +- .../hadoop/yarn/util/resource/Resources.java | 51 +- .../yarn/util/timeline/TimelineUtils.java | 49 + .../org/apache/hadoop/yarn/webapp/Router.java | 2 +- .../hadoop/yarn/webapp/util/WebAppUtils.java | 59 +- .../src/main/resources/webapps/timeline/.keep | 0 .../src/main/resources/yarn-default.xml | 283 +- .../hadoop/yarn/TestContainerLaunchRPC.java | 16 +- .../yarn/TestContainerResourceIncreaseRPC.java | 16 +- .../java/org/apache/hadoop/yarn/TestRPC.java | 265 - .../hadoop/yarn/api/TestPBImplRecords.java | 8 + .../TestTimelineServiceRecords.java | 312 + .../client/api/impl/TestTimelineClient.java | 51 +- .../api/impl/TestTimelineClientForATS1_5.java | 4 +- .../api/impl/TestTimelineClientV2Impl.java | 378 + .../hadoop/yarn/event/DrainDispatcher.java | 58 +- .../logaggregation/TestAggregatedLogFormat.java | 5 +- .../yarn/util/TestTimelineServiceHelper.java | 83 + .../yarn/util/TestWindowsBasedProcessTree.java | 7 +- .../yarn/util/resource/TestResources.java | 6 +- .../hadoop/yarn/webapp/GuiceServletConfig.java | 43 + .../yarn/webapp/WebServicesTestUtils.java | 11 + .../yarn/webapp/util/TestWebAppUtils.java | 42 + .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 10 +- .../pom.xml | 4 +- .../webapp/AHSWebServices.java | 15 +- .../timeline/RollingLevelDBTimelineStore.java | 39 +- .../timeline/webapp/TimelineWebServices.java | 2 - .../webapp/TestAHSWebServices.java | 67 +- .../webapp/TestTimelineWebServices.java | 56 +- .../hadoop-yarn-server-common/pom.xml | 7 +- .../api/CollectorNodemanagerProtocol.java | 73 + .../api/CollectorNodemanagerProtocolPB.java | 34 + .../api/DistributedSchedulerProtocol.java | 78 - .../api/DistributedSchedulerProtocolPB.java | 36 - .../api/DistributedSchedulingAMProtocol.java | 87 + .../api/DistributedSchedulingAMProtocolPB.java | 37 + .../hadoop/yarn/server/api/ServerRMProxy.java | 2 +- ...ollectorNodemanagerProtocolPBClientImpl.java | 114 + ...istributedSchedulerProtocolPBClientImpl.java | 151 - ...ributedSchedulingAMProtocolPBClientImpl.java | 153 + ...llectorNodemanagerProtocolPBServiceImpl.java | 82 + ...stributedSchedulerProtocolPBServiceImpl.java | 143 - ...ibutedSchedulingAMProtocolPBServiceImpl.java | 151 + .../DistSchedAllocateResponse.java | 58 - .../DistSchedRegisterResponse.java | 102 - .../DistributedSchedulingAllocateRequest.java | 71 + .../DistributedSchedulingAllocateResponse.java | 66 + .../GetTimelineCollectorContextRequest.java | 37 + .../GetTimelineCollectorContextResponse.java | 51 + .../protocolrecords/NodeHeartbeatRequest.java | 23 + .../protocolrecords/NodeHeartbeatResponse.java | 4 + ...RegisterDistributedSchedulingAMResponse.java | 108 + .../ReportNewCollectorInfoRequest.java | 53 + .../ReportNewCollectorInfoResponse.java | 32 + .../pb/DistSchedAllocateResponsePBImpl.java | 180 - .../pb/DistSchedRegisterResponsePBImpl.java | 308 - ...tributedSchedulingAllocateRequestPBImpl.java | 187 + ...ributedSchedulingAllocateResponsePBImpl.java | 190 + ...etTimelineCollectorContextRequestPBImpl.java | 132 + ...tTimelineCollectorContextResponsePBImpl.java | 159 + .../impl/pb/NodeHeartbeatRequestPBImpl.java | 62 + .../impl/pb/NodeHeartbeatResponsePBImpl.java | 47 + ...erDistributedSchedulingAMResponsePBImpl.java | 332 + .../pb/ReportNewCollectorInfoRequestPBImpl.java | 144 + .../ReportNewCollectorInfoResponsePBImpl.java | 76 + .../server/api/records/AppCollectorsMap.java | 46 + .../records/impl/pb/AppCollectorsMapPBImpl.java | 152 + .../metrics/ContainerMetricsConstants.java | 18 + .../OpportunisticContainerAllocator.java | 378 + .../OpportunisticContainerContext.java | 178 + .../yarn/server/scheduler/package-info.java | 22 + .../hadoop/yarn/server/utils/BuilderUtils.java | 23 +- .../proto/collectornodemanager_protocol.proto | 30 + .../proto/distributed_scheduler_protocol.proto | 39 - .../distributed_scheduling_am_protocol.proto | 39 + .../yarn_server_common_service_protos.proto | 46 +- .../java/org/apache/hadoop/yarn/TestRPC.java | 416 + .../hadoop/yarn/TestYarnServerApiClasses.java | 19 +- .../hadoop-yarn-server-nodemanager/pom.xml | 16 +- .../src/CMakeLists.txt | 29 +- .../src/config.h.cmake | 20 +- .../server/nodemanager/ContainerExecutor.java | 467 +- .../hadoop/yarn/server/nodemanager/Context.java | 17 +- .../nodemanager/DefaultContainerExecutor.java | 1 - .../server/nodemanager/DirectoryCollection.java | 243 +- .../nodemanager/DockerContainerExecutor.java | 1 - .../nodemanager/LinuxContainerExecutor.java | 89 +- .../yarn/server/nodemanager/NodeManager.java | 95 +- .../nodemanager/NodeStatusUpdaterImpl.java | 47 +- .../WindowsSecureContainerExecutor.java | 12 +- .../nodemanager/amrmproxy/AMRMProxyService.java | 7 +- .../amrmproxy/AbstractRequestInterceptor.java | 22 +- .../amrmproxy/DefaultRequestInterceptor.java | 97 +- .../amrmproxy/RequestInterceptor.java | 4 +- .../collectormanager/NMCollectorService.java | 145 + .../collectormanager/package-info.java | 28 + .../containermanager/ContainerManagerImpl.java | 113 +- .../application/Application.java | 5 + .../ApplicationContainerFinishedEvent.java | 18 +- .../application/ApplicationImpl.java | 91 +- .../containermanager/container/Container.java | 3 + .../container/ContainerImpl.java | 23 +- .../launcher/ContainerLaunch.java | 46 +- .../runtime/DefaultLinuxContainerRuntime.java | 20 +- .../DelegatingLinuxContainerRuntime.java | 14 +- .../runtime/DockerLinuxContainerRuntime.java | 205 +- .../linux/runtime/LinuxContainerRuntime.java | 10 +- .../runtime/LinuxContainerRuntimeConstants.java | 4 + .../runtime/docker/DockerInspectCommand.java | 41 + .../linux/runtime/docker/DockerRmCommand.java | 30 + .../linux/runtime/docker/DockerStopCommand.java | 39 + .../localizer/ContainerLocalizer.java | 13 +- .../localizer/ResourceLocalizationService.java | 19 +- .../monitor/ContainerMetrics.java | 2 + .../monitor/ContainersMonitorImpl.java | 32 +- .../runtime/ContainerRuntime.java | 40 +- .../executor/ContainerStartContext.java | 29 +- .../scheduler/DistributedScheduler.java | 272 + .../nodemanager/scheduler/LocalScheduler.java | 419 - .../OpportunisticContainerAllocator.java | 187 - .../timelineservice/NMTimelineEvent.java | 35 + .../timelineservice/NMTimelineEventType.java | 27 + .../timelineservice/NMTimelinePublisher.java | 405 + .../timelineservice/package-info.java | 29 + .../nodemanager/webapp/NMWebAppFilter.java | 10 +- .../nodemanager/webapp/NMWebServices.java | 75 +- .../webapp/dao/ContainerLogsInfo.java | 112 + .../container-executor/impl/compat/fchmodat.h | 56 + .../container-executor/impl/compat/fdopendir.h | 52 + .../container-executor/impl/compat/fstatat.h | 67 + .../container-executor/impl/compat/openat.h | 74 + .../container-executor/impl/compat/unlinkat.h | 67 + .../container-executor/impl/configuration.c | 15 +- .../container-executor/impl/configuration.h | 10 +- .../impl/container-executor.c | 182 +- .../impl/container-executor.h | 20 +- .../container-executor/impl/get_executable.c | 215 + .../main/native/container-executor/impl/main.c | 38 +- .../test/test-container-executor.c | 149 +- .../nodemanager/TestContainerExecutor.java | 11 +- .../yarn/server/nodemanager/TestEventFlow.java | 2 +- .../TestLinuxContainerExecutorWithMocks.java | 21 +- .../nodemanager/TestNodeStatusUpdater.java | 15 +- .../amrmproxy/BaseAMRMProxyTest.java | 24 +- .../BaseContainerManagerTest.java | 3 +- .../TestContainerManagerRecovery.java | 13 +- .../application/TestApplication.java | 13 +- .../launcher/TestContainerLaunch.java | 41 +- .../runtime/TestDockerContainerRuntime.java | 155 +- .../docker/TestDockerInspectCommand.java | 50 + .../runtime/docker/TestDockerRmCommand.java | 48 + .../runtime/docker/TestDockerStopCommand.java | 55 + .../TestLocalCacheDirectoryManager.java | 2 +- .../TestResourceLocalizationService.java | 5 +- .../TestAppLogAggregatorImpl.java | 4 +- .../TestLogAggregationService.java | 2 +- .../metrics/TestNodeManagerMetrics.java | 8 +- .../scheduler/TestDistributedScheduler.java | 268 + .../scheduler/TestLocalScheduler.java | 222 - .../TestNMTimelinePublisher.java | 157 + .../yarn/server/nodemanager/webapp/MockApp.java | 22 + .../nodemanager/webapp/MockContainer.java | 5 + .../webapp/TestContainerLogsPage.java | 15 +- .../nodemanager/webapp/TestNMAppsPage.java | 2 +- .../nodemanager/webapp/TestNMWebServer.java | 8 +- .../nodemanager/webapp/TestNMWebServices.java | 104 +- .../webapp/TestNMWebServicesApps.java | 44 +- .../webapp/TestNMWebServicesContainers.java | 31 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 14 +- .../server/resourcemanager/AdminService.java | 239 +- .../ApplicationMasterService.java | 21 +- .../server/resourcemanager/ClientRMService.java | 49 +- .../DecommissioningNodesWatcher.java | 439 + .../DistributedSchedulingService.java | 335 - .../resourcemanager/NodesListManager.java | 166 +- ...pportunisticContainerAllocatorAMService.java | 367 + .../resourcemanager/RMActiveServiceContext.java | 20 +- .../server/resourcemanager/RMAppManager.java | 42 +- .../server/resourcemanager/RMAuditLogger.java | 40 +- .../yarn/server/resourcemanager/RMContext.java | 6 + .../server/resourcemanager/RMContextImpl.java | 19 +- .../server/resourcemanager/RMServerUtils.java | 3 +- .../server/resourcemanager/ResourceManager.java | 95 +- .../resourcemanager/ResourceTrackerService.java | 90 +- .../resourcemanager/amlauncher/AMLauncher.java | 58 +- .../metrics/AbstractSystemMetricsPublisher.java | 178 + .../metrics/AppAttemptFinishedEvent.java | 90 - .../metrics/AppAttemptRegisteredEvent.java | 81 - .../metrics/ApplicaitonStateUpdatedEvent.java | 47 - .../metrics/ApplicationACLsUpdatedEvent.java | 45 - .../metrics/ApplicationCreatedEvent.java | 124 - .../metrics/ApplicationFinishedEvent.java | 82 - .../metrics/ApplicationUpdatedEvent.java | 54 - .../metrics/ContainerCreatedEvent.java | 73 - .../metrics/ContainerFinishedEvent.java | 72 - .../metrics/NoOpSystemMetricPublisher.java | 70 + .../metrics/SystemMetricsEvent.java | 33 - .../metrics/SystemMetricsEventType.java | 37 - .../metrics/SystemMetricsPublisher.java | 608 +- .../metrics/TimelineServiceV1Publisher.java | 386 + .../metrics/TimelineServiceV2Publisher.java | 449 + .../resourcemanager/metrics/package-info.java | 28 + .../capacity/FifoCandidatesSelector.java | 11 +- .../reservation/CapacityOverTimePolicy.java | 299 +- .../resourcemanager/resource/Priority.java | 10 +- .../server/resourcemanager/rmapp/RMApp.java | 23 + .../rmapp/RMAppCollectorUpdateEvent.java | 40 + .../resourcemanager/rmapp/RMAppEventType.java | 3 + .../server/resourcemanager/rmapp/RMAppImpl.java | 135 +- .../rmapp/RMAppKillByClientEvent.java | 67 + .../rmapp/attempt/RMAppAttemptImpl.java | 44 +- .../rmcontainer/RMContainer.java | 18 +- .../rmcontainer/RMContainerImpl.java | 50 +- .../rmcontainer/RMContainerReservedEvent.java | 12 +- .../server/resourcemanager/rmnode/RMNode.java | 6 + .../rmnode/RMNodeDecommissioningEvent.java | 41 + .../resourcemanager/rmnode/RMNodeImpl.java | 54 +- .../scheduler/AbstractYarnScheduler.java | 26 +- .../scheduler/AppSchedulingInfo.java | 165 +- .../scheduler/ClusterNodeTracker.java | 58 +- .../resourcemanager/scheduler/QueueMetrics.java | 27 +- .../scheduler/SchedulerApplicationAttempt.java | 182 +- .../scheduler/SchedulerNode.java | 5 +- .../scheduler/SchedulerRequestKey.java | 122 + .../scheduler/activities/ActivitiesLogger.java | 275 + .../scheduler/activities/ActivitiesManager.java | 319 + .../activities/ActivityDiagnosticConstant.java | 77 + .../scheduler/activities/ActivityNode.java | 110 + .../scheduler/activities/ActivityState.java | 37 + .../activities/AllocationActivity.java | 74 + .../scheduler/activities/AllocationState.java | 35 + .../scheduler/activities/AppAllocation.java | 107 + .../scheduler/activities/NodeAllocation.java | 139 + .../scheduler/capacity/AbstractCSQueue.java | 9 +- .../scheduler/capacity/CSAssignment.java | 31 +- .../scheduler/capacity/CSQueueUtils.java | 55 +- .../capacity/CapacityHeadroomProvider.java | 2 +- .../scheduler/capacity/CapacityScheduler.java | 41 +- .../capacity/CapacitySchedulerContext.java | 3 + .../scheduler/capacity/LeafQueue.java | 86 +- .../scheduler/capacity/ParentQueue.java | 134 +- .../allocator/AbstractContainerAllocator.java | 62 +- .../capacity/allocator/ContainerAllocator.java | 10 +- .../allocator/IncreaseContainerAllocator.java | 35 +- .../allocator/RegularContainerAllocator.java | 294 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 76 +- .../common/fica/FiCaSchedulerNode.java | 5 +- .../scheduler/fair/AllocationConfiguration.java | 87 +- .../fair/AllocationFileLoaderService.java | 96 +- .../scheduler/fair/FSAppAttempt.java | 224 +- .../scheduler/fair/FSLeafQueue.java | 33 +- .../resourcemanager/scheduler/fair/FSQueue.java | 19 - .../scheduler/fair/FSSchedulerNode.java | 5 +- .../scheduler/fair/FairScheduler.java | 33 +- .../scheduler/fair/QueueManager.java | 168 +- .../scheduler/fair/SchedulingPolicy.java | 11 - .../fair/policies/ComputeFairShares.java | 4 +- .../DominantResourceFairnessPolicy.java | 5 - .../fair/policies/FairSharePolicy.java | 5 - .../scheduler/fair/policies/FifoPolicy.java | 5 - .../scheduler/fifo/FifoScheduler.java | 92 +- .../security/DelegationTokenRenewer.java | 7 +- .../security/QueueACLsManager.java | 12 +- .../RMTimelineCollectorManager.java | 104 + .../timelineservice/package-info.java | 28 + .../webapp/RMAppAttemptBlock.java | 13 +- .../resourcemanager/webapp/RMWebAppFilter.java | 24 +- .../resourcemanager/webapp/RMWebServices.java | 132 +- .../webapp/dao/ActivitiesInfo.java | 80 + .../webapp/dao/ActivityNodeInfo.java | 67 + .../webapp/dao/AppActivitiesInfo.java | 79 + .../webapp/dao/AppAllocationInfo.java | 72 + .../resourcemanager/webapp/dao/AppInfo.java | 40 +- .../webapp/dao/ClusterMetricsInfo.java | 2 +- .../webapp/dao/ExecutionTypeRequestInfo.java | 63 + .../webapp/dao/NodeAllocationInfo.java | 51 + .../dao/PartitionQueueCapacitiesInfo.java | 4 +- .../webapp/dao/QueueCapacitiesInfo.java | 4 +- .../webapp/dao/ResourceInfo.java | 8 +- .../webapp/dao/ResourceRequestInfo.java | 131 + .../server/resourcemanager/Application.java | 120 +- .../yarn/server/resourcemanager/MockAM.java | 44 +- .../yarn/server/resourcemanager/MockNodes.java | 14 +- .../yarn/server/resourcemanager/MockRM.java | 87 +- .../ParameterizedSchedulerTestBase.java | 20 + .../yarn/server/resourcemanager/Task.java | 11 +- .../server/resourcemanager/TestAppManager.java | 48 + .../resourcemanager/TestApplicationACLs.java | 3 +- .../resourcemanager/TestClientRMService.java | 34 +- .../TestDecommissioningNodesWatcher.java | 131 + .../TestDistributedSchedulingService.java | 259 - ...pportunisticContainerAllocatorAMService.java | 271 + .../yarn/server/resourcemanager/TestRM.java | 2 - .../resourcemanager/TestRMAdminService.java | 1 + .../resourcemanager/TestRMAuditLogger.java | 63 +- .../resourcemanager/TestRMNodeTransitions.java | 11 - .../server/resourcemanager/TestRMRestart.java | 84 +- .../TestResourceTrackerService.java | 291 +- .../TestSubmitApplicationWithRMHA.java | 4 +- .../TestWorkPreservingRMRestart.java | 2 +- .../applicationsmanager/MockAsm.java | 12 + .../applicationsmanager/TestAMRestart.java | 11 +- .../TestRMAppLogAggregationStatus.java | 6 +- .../metrics/TestSystemMetricsPublisher.java | 6 +- .../TestSystemMetricsPublisherForV2.java | 398 + ...alCapacityPreemptionPolicyMockFramework.java | 4 + ...estProportionalCapacityPreemptionPolicy.java | 4 + .../reservation/TestCapacityOverTimePolicy.java | 20 +- .../reservation/TestReservationSystemUtil.java | 2 +- .../resourcetracker/TestNMReconnect.java | 52 +- .../server/resourcemanager/rmapp/MockRMApp.java | 14 + .../rmapp/TestRMAppTransitions.java | 94 +- .../attempt/TestRMAppAttemptTransitions.java | 7 + .../rmcontainer/TestRMContainerImpl.java | 6 +- .../scheduler/TestAppSchedulingInfo.java | 37 +- .../scheduler/TestClusterNodeTracker.java | 69 + .../scheduler/TestQueueMetrics.java | 6 +- .../TestSchedulerApplicationAttempt.java | 31 +- .../TestSchedulingWithAllocationRequestId.java | 274 + .../capacity/TestApplicationPriority.java | 2 + .../capacity/TestCapacityScheduler.java | 343 +- .../scheduler/capacity/TestLeafQueue.java | 170 +- .../TestNodeLabelContainerAllocation.java | 238 +- .../scheduler/capacity/TestReservations.java | 52 +- .../scheduler/capacity/TestUtils.java | 18 + .../fair/TestAllocationFileLoaderService.java | 71 +- .../fair/TestContinuousScheduling.java | 4 +- .../scheduler/fair/TestFSAppAttempt.java | 18 +- .../scheduler/fair/TestFairScheduler.java | 364 +- .../fair/TestFairSchedulerPreemption.java | 24 +- .../scheduler/fair/TestQueueManager.java | 169 +- .../security/TestDelegationTokenRenewer.java | 7 +- .../security/TestRMDelegationTokens.java | 36 +- .../webapp/TestRMWebServices.java | 29 +- .../webapp/TestRMWebServicesApps.java | 169 +- .../TestRMWebServicesAppsModification.java | 89 +- .../webapp/TestRMWebServicesCapacitySched.java | 67 +- .../TestRMWebServicesDelegationTokens.java | 66 +- .../webapp/TestRMWebServicesFairScheduler.java | 29 +- .../TestRMWebServicesForCSWithPartitions.java | 94 +- .../webapp/TestRMWebServicesNodeLabels.java | 30 +- .../webapp/TestRMWebServicesNodes.java | 35 +- .../webapp/TestRMWebServicesReservation.java | 139 +- .../TestRMWebServicesSchedulerActivities.java | 777 ++ .../yarn/webapp/TestRMWithCSRFFilter.java | 22 +- .../hadoop/yarn/webapp/TestRMWithXFSFilter.java | 19 +- .../pom.xml | 4 +- .../hadoop-yarn-server-tests/pom.xml | 21 +- .../hadoop/yarn/server/MiniYARNCluster.java | 28 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 2 - .../TestTimelineServiceClientIntegration.java | 188 + .../pom.xml | 4 +- .../yarn/server/timeline/EntityCacheItem.java | 81 +- .../timeline/EntityGroupFSTimelineStore.java | 137 +- .../TestEntityGroupFSTimelineStore.java | 161 +- .../pom.xml | 457 + ...stTimelineReaderWebServicesHBaseStorage.java | 2166 ++++ .../storage/TestHBaseTimelineStorage.java | 3751 +++++++ ...TestPhoenixOfflineAggregationWriterImpl.java | 161 + .../storage/flow/TestFlowDataGenerator.java | 386 + .../flow/TestHBaseStorageFlowActivity.java | 483 + .../storage/flow/TestHBaseStorageFlowRun.java | 1034 ++ .../flow/TestHBaseStorageFlowRunCompaction.java | 831 ++ .../src/test/resources/log4j.properties | 19 + .../hadoop-yarn-server-timelineservice/pom.xml | 239 + .../server/timelineservice/TimelineContext.java | 146 + .../collector/AppLevelTimelineCollector.java | 161 + .../collector/NodeTimelineCollectorManager.java | 223 + .../PerNodeTimelineCollectorsAuxService.java | 231 + .../collector/TimelineCollector.java | 341 + .../collector/TimelineCollectorContext.java | 76 + .../collector/TimelineCollectorManager.java | 254 + .../collector/TimelineCollectorWebService.java | 250 + .../timelineservice/collector/package-info.java | 29 + .../server/timelineservice/package-info.java | 28 + .../reader/TimelineDataToRetrieve.java | 147 + .../reader/TimelineEntityFilters.java | 242 + .../reader/TimelineParseConstants.java | 34 + .../reader/TimelineParseException.java | 36 + .../timelineservice/reader/TimelineParser.java | 37 + .../reader/TimelineParserForCompareExpr.java | 300 + .../reader/TimelineParserForDataToRetrieve.java | 95 + .../reader/TimelineParserForEqualityExpr.java | 343 + .../reader/TimelineParserForExistFilters.java | 51 + .../reader/TimelineParserForKVFilters.java | 78 + .../reader/TimelineParserForNumericFilters.java | 72 + .../TimelineParserForRelationFilters.java | 71 + .../reader/TimelineReaderContext.java | 98 + .../reader/TimelineReaderManager.java | 179 + .../reader/TimelineReaderServer.java | 178 + .../reader/TimelineReaderUtils.java | 171 + .../reader/TimelineReaderWebServices.java | 2123 ++++ .../reader/TimelineReaderWebServicesUtils.java | 247 + .../reader/TimelineUIDConverter.java | 247 + .../reader/filter/TimelineCompareFilter.java | 147 + .../reader/filter/TimelineCompareOp.java | 36 + .../reader/filter/TimelineExistsFilter.java | 107 + .../reader/filter/TimelineFilter.java | 68 + .../reader/filter/TimelineFilterList.java | 141 + .../reader/filter/TimelineFilterUtils.java | 290 + .../reader/filter/TimelineKeyValueFilter.java | 61 + .../reader/filter/TimelineKeyValuesFilter.java | 126 + .../reader/filter/TimelinePrefixFilter.java | 99 + .../reader/filter/package-info.java | 28 + .../timelineservice/reader/package-info.java | 29 + .../storage/FileSystemTimelineReaderImpl.java | 406 + .../storage/FileSystemTimelineWriterImpl.java | 160 + .../storage/HBaseTimelineReaderImpl.java | 88 + .../storage/HBaseTimelineWriterImpl.java | 571 ++ .../storage/OfflineAggregationWriter.java | 67 + .../PhoenixOfflineAggregationWriterImpl.java | 358 + .../storage/TimelineAggregationTrack.java | 28 + .../timelineservice/storage/TimelineReader.java | 180 + .../storage/TimelineSchemaCreator.java | 272 + .../timelineservice/storage/TimelineWriter.java | 87 + .../storage/application/ApplicationColumn.java | 156 + .../application/ApplicationColumnFamily.java | 65 + .../application/ApplicationColumnPrefix.java | 288 + .../storage/application/ApplicationRowKey.java | 206 + .../application/ApplicationRowKeyPrefix.java | 69 + .../storage/application/ApplicationTable.java | 161 + .../storage/application/package-info.java | 28 + .../storage/apptoflow/AppToFlowColumn.java | 148 + .../apptoflow/AppToFlowColumnFamily.java | 51 + .../storage/apptoflow/AppToFlowRowKey.java | 143 + .../storage/apptoflow/AppToFlowTable.java | 113 + .../storage/apptoflow/package-info.java | 28 + .../storage/common/AppIdKeyConverter.java | 95 + .../storage/common/BaseTable.java | 140 + .../common/BufferedMutatorDelegator.java | 73 + .../timelineservice/storage/common/Column.java | 80 + .../storage/common/ColumnFamily.java | 34 + .../storage/common/ColumnHelper.java | 388 + .../storage/common/ColumnPrefix.java | 145 + .../storage/common/EventColumnName.java | 63 + .../common/EventColumnNameConverter.java | 99 + .../storage/common/GenericConverter.java | 48 + .../storage/common/KeyConverter.java | 41 + .../storage/common/LongConverter.java | 94 + .../storage/common/LongKeyConverter.java | 68 + .../storage/common/NumericValueConverter.java | 39 + .../storage/common/OfflineAggregationInfo.java | 115 + .../timelineservice/storage/common/Range.java | 62 + .../storage/common/RowKeyPrefix.java | 42 + .../storage/common/Separator.java | 575 ++ .../storage/common/StringKeyConverter.java | 54 + .../common/TimelineEntityFiltersType.java | 71 + .../common/TimelineHBaseSchemaConstants.java | 71 + .../storage/common/TimelineStorageUtils.java | 586 ++ .../storage/common/TimestampGenerator.java | 116 + .../storage/common/TypedBufferedMutator.java | 28 + .../storage/common/ValueConverter.java | 47 + .../storage/common/package-info.java | 28 + .../storage/entity/EntityColumn.java | 160 + .../storage/entity/EntityColumnFamily.java | 65 + .../storage/entity/EntityColumnPrefix.java | 300 + .../storage/entity/EntityRowKey.java | 225 + .../storage/entity/EntityRowKeyPrefix.java | 74 + .../storage/entity/EntityTable.java | 161 + .../storage/entity/package-info.java | 28 + .../flow/AggregationCompactionDimension.java | 63 + .../storage/flow/AggregationOperation.java | 94 + .../timelineservice/storage/flow/Attribute.java | 39 + .../storage/flow/FlowActivityColumnFamily.java | 55 + .../storage/flow/FlowActivityColumnPrefix.java | 277 + .../storage/flow/FlowActivityRowKey.java | 196 + .../storage/flow/FlowActivityRowKeyPrefix.java | 60 + .../storage/flow/FlowActivityTable.java | 108 + .../storage/flow/FlowRunColumn.java | 182 + .../storage/flow/FlowRunColumnFamily.java | 54 + .../storage/flow/FlowRunColumnPrefix.java | 268 + .../storage/flow/FlowRunCoprocessor.java | 304 + .../storage/flow/FlowRunRowKey.java | 190 + .../storage/flow/FlowRunRowKeyPrefix.java | 54 + .../storage/flow/FlowRunTable.java | 141 + .../storage/flow/FlowScanner.java | 727 ++ .../storage/flow/FlowScannerOperation.java | 46 + .../storage/flow/package-info.java | 29 + .../timelineservice/storage/package-info.java | 28 + .../storage/reader/ApplicationEntityReader.java | 481 + .../reader/FlowActivityEntityReader.java | 163 + .../storage/reader/FlowRunEntityReader.java | 269 + .../storage/reader/GenericEntityReader.java | 648 ++ .../storage/reader/TimelineEntityReader.java | 496 + .../reader/TimelineEntityReaderFactory.java | 89 + .../storage/reader/package-info.java | 28 + .../TestAppLevelTimelineCollector.java | 23 + .../TestNMTimelineCollectorManager.java | 170 + ...TestPerNodeTimelineCollectorsAuxService.java | 203 + .../collector/TestTimelineCollector.java | 127 + .../reader/TestTimelineReaderServer.java | 57 + .../reader/TestTimelineReaderUtils.java | 55 + .../reader/TestTimelineReaderWebServices.java | 570 ++ .../TestTimelineReaderWebServicesUtils.java | 923 ++ .../reader/TestTimelineUIDConverter.java | 97 + .../TestFileSystemTimelineReaderImpl.java | 813 ++ .../TestFileSystemTimelineWriterImpl.java | 135 + .../storage/common/TestKeyConverters.java | 130 + .../storage/common/TestRowKeys.java | 246 + .../storage/common/TestSeparator.java | 215 + .../src/test/resources/log4j.properties | 19 + .../hadoop-yarn-server-web-proxy/pom.xml | 4 +- .../server/webproxy/amfilter/TestAmFilter.java | 2 +- .../hadoop-yarn/hadoop-yarn-server/pom.xml | 6 +- .../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +- .../src/site/markdown/CapacityScheduler.md | 16 +- .../src/site/markdown/FairScheduler.md | 3 + .../src/site/markdown/ResourceManagerRest.md | 7 + .../src/site/markdown/TimelineServer.md | 2 +- .../src/site/markdown/TimelineServiceV2.md | 1199 +++ .../src/site/markdown/YarnCommands.md | 2 +- .../site/resources/images/flow_hierarchy.png | Bin 0 -> 42345 bytes .../src/site/resources/images/timeline_v2.jpg | Bin 0 -> 45112 bytes hadoop-yarn-project/hadoop-yarn/pom.xml | 6 +- hadoop-yarn-project/pom.xml | 4 +- pom.xml | 30 +- 1452 files changed, 110483 insertions(+), 20124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db22affd/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db22affd/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db22affd/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db22affd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
