Merge remote-tracking branch 'origin/trunk' into MR-2841
Conflicts:
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cce7d1e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cce7d1e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cce7d1e2
Branch: refs/heads/HDFS-6584
Commit: cce7d1e2f96ff75fe19ab2879ddd90a898ca5b18
Parents: 7ade9b0 727331b
Author: Todd Lipcon <[email protected]>
Authored: Tue Sep 2 15:55:54 2014 -0700
Committer: Todd Lipcon <[email protected]>
Committed: Tue Sep 2 15:55:54 2014 -0700
----------------------------------------------------------------------
.gitignore | 1 +
BUILDING.txt | 22 +
dev-support/test-patch.sh | 26 +-
.../main/resources/assemblies/hadoop-dist.xml | 7 +
.../dev-support/findbugsExcludeFile.xml | 38 +
hadoop-common-project/hadoop-auth/pom.xml | 26 +
.../authentication/client/AuthenticatedURL.java | 36 +-
.../server/AuthenticationFilter.java | 109 +-
.../server/KerberosAuthenticationHandler.java | 21 +-
.../server/PseudoAuthenticationHandler.java | 21 +-
.../util/RandomSignerSecretProvider.java | 49 +
.../util/RolloverSignerSecretProvider.java | 139 ++
.../security/authentication/util/Signer.java | 46 +-
.../util/SignerSecretProvider.java | 62 +
.../util/StringSignerSecretProvider.java | 49 +
.../client/AuthenticatorTestCase.java | 137 +-
.../client/TestAuthenticatedURL.java | 38 +-
.../client/TestKerberosAuthenticator.java | 58 +-
.../server/TestAuthenticationFilter.java | 118 +-
.../util/TestRandomSignerSecretProvider.java | 63 +
.../util/TestRolloverSignerSecretProvider.java | 79 +
.../authentication/util/TestSigner.java | 85 +-
.../util/TestStringSignerSecretProvider.java | 33 +
hadoop-common-project/hadoop-common/CHANGES.txt | 502 +++++--
.../dev-support/findbugsExcludeFile.xml | 5 +
hadoop-common-project/hadoop-common/pom.xml | 34 +-
.../hadoop-common/src/CMakeLists.txt | 34 +
.../hadoop-common/src/JNIFlags.cmake | 6 +
.../hadoop-common/src/config.h.cmake | 1 +
.../src/contrib/bash-tab-completion/hadoop.sh | 28 +-
.../hadoop-common/src/main/bin/hadoop | 229 +--
.../src/main/bin/hadoop-config.cmd | 10 +-
.../hadoop-common/src/main/bin/hadoop-config.sh | 409 ++---
.../hadoop-common/src/main/bin/hadoop-daemon.sh | 214 +--
.../src/main/bin/hadoop-daemons.sh | 37 +-
.../src/main/bin/hadoop-functions.sh | 1066 ++++++++++++++
.../src/main/bin/hadoop-layout.sh.example | 93 ++
.../hadoop-common/src/main/bin/hadoop.cmd | 33 +-
.../hadoop-common/src/main/bin/rcc | 52 +-
.../hadoop-common/src/main/bin/slaves.sh | 51 +-
.../hadoop-common/src/main/bin/start-all.sh | 38 +-
.../hadoop-common/src/main/bin/stop-all.sh | 36 +-
.../hadoop-common/src/main/conf/hadoop-env.sh | 432 +++++-
.../org/apache/hadoop/conf/Configuration.java | 78 +-
.../hadoop/conf/ReconfigurationServlet.java | 2 +
.../apache/hadoop/crypto/AesCtrCryptoCodec.java | 67 +
.../org/apache/hadoop/crypto/CipherSuite.java | 115 ++
.../org/apache/hadoop/crypto/CryptoCodec.java | 179 +++
.../apache/hadoop/crypto/CryptoInputStream.java | 680 +++++++++
.../hadoop/crypto/CryptoOutputStream.java | 280 ++++
.../apache/hadoop/crypto/CryptoStreamUtils.java | 70 +
.../org/apache/hadoop/crypto/Decryptor.java | 72 +
.../org/apache/hadoop/crypto/Encryptor.java | 71 +
.../hadoop/crypto/JceAesCtrCryptoCodec.java | 165 +++
.../hadoop/crypto/OpensslAesCtrCryptoCodec.java | 164 +++
.../org/apache/hadoop/crypto/OpensslCipher.java | 289 ++++
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 248 +++-
.../apache/hadoop/crypto/key/KeyProvider.java | 32 +-
.../crypto/key/KeyProviderCryptoExtension.java | 91 +-
.../KeyProviderDelegationTokenExtension.java | 8 +-
.../hadoop/crypto/key/KeyProviderExtension.java | 1 +
.../org/apache/hadoop/crypto/key/KeyShell.java | 89 +-
.../apache/hadoop/crypto/key/UserProvider.java | 5 +-
.../crypto/key/kms/KMSClientProvider.java | 117 +-
.../hadoop/crypto/key/kms/KMSRESTConstants.java | 2 +-
.../crypto/random/OpensslSecureRandom.java | 122 ++
.../hadoop/crypto/random/OsSecureRandom.java | 115 ++
.../apache/hadoop/fs/AbstractFileSystem.java | 103 +-
.../apache/hadoop/fs/ChecksumFileSystem.java | 8 +-
.../java/org/apache/hadoop/fs/ChecksumFs.java | 8 +-
.../hadoop/fs/CommonConfigurationKeys.java | 4 +-
.../fs/CommonConfigurationKeysPublic.java | 34 +-
.../org/apache/hadoop/fs/ContentSummary.java | 46 +-
.../apache/hadoop/fs/FSDataOutputStream.java | 2 +-
.../org/apache/hadoop/fs/FSOutputSummer.java | 107 +-
.../java/org/apache/hadoop/fs/FileContext.java | 141 +-
.../apache/hadoop/fs/FileEncryptionInfo.java | 102 ++
.../java/org/apache/hadoop/fs/FileSystem.java | 159 +-
.../org/apache/hadoop/fs/FilterFileSystem.java | 7 +
.../java/org/apache/hadoop/fs/FilterFs.java | 8 +
.../main/java/org/apache/hadoop/fs/Globber.java | 8 +-
.../fs/crypto/CryptoFSDataInputStream.java | 37 +
.../fs/crypto/CryptoFSDataOutputStream.java | 47 +
.../hadoop/fs/shell/CommandWithDestination.java | 75 +-
.../apache/hadoop/fs/shell/CopyCommands.java | 6 +-
.../java/org/apache/hadoop/fs/shell/Count.java | 36 +-
.../java/org/apache/hadoop/fs/shell/Delete.java | 6 +-
.../org/apache/hadoop/fs/shell/Display.java | 12 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 8 +
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 7 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 10 +-
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 9 +
.../org/apache/hadoop/http/HttpServer2.java | 16 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 9 +-
.../org/apache/hadoop/io/nativeio/NativeIO.java | 19 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 8 +
.../apache/hadoop/ipc/DecayRpcScheduler.java | 522 +++++++
.../hadoop/ipc/DecayRpcSchedulerMXBean.java | 30 +
.../org/apache/hadoop/ipc/FairCallQueue.java | 449 ++++++
.../apache/hadoop/ipc/FairCallQueueMXBean.java | 27 +
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 20 +-
.../org/apache/hadoop/ipc/RpcMultiplexer.java | 32 +
.../org/apache/hadoop/ipc/RpcScheduler.java | 29 +
.../main/java/org/apache/hadoop/ipc/Server.java | 43 +-
.../ipc/WeightedRoundRobinMultiplexer.java | 2 +-
.../apache/hadoop/ipc/WritableRpcEngine.java | 18 +-
.../apache/hadoop/ipc/metrics/RpcMetrics.java | 8 +-
.../org/apache/hadoop/jmx/JMXJsonServlet.java | 9 +-
.../metrics2/impl/MetricsCollectorImpl.java | 6 +-
.../apache/hadoop/metrics2/lib/MutableStat.java | 8 +
.../net/NetworkTopologyWithNodeGroup.java | 2 +-
.../JniBasedUnixGroupsMappingWithFallback.java | 3 +-
.../hadoop/security/LdapGroupsMapping.java | 26 +-
.../apache/hadoop/security/NetgroupCache.java | 17 +-
.../apache/hadoop/security/SecurityUtil.java | 3 +-
.../hadoop/security/WhitelistBasedResolver.java | 149 ++
.../hadoop/security/alias/CredentialShell.java | 65 +-
.../hadoop/security/alias/UserProvider.java | 5 -
.../authorize/ServiceAuthorizationManager.java | 38 +-
.../security/ssl/FileBasedKeyStoresFactory.java | 22 +-
.../web/DelegationTokenAuthenticatedURL.java | 398 +++++
.../DelegationTokenAuthenticationFilter.java | 274 ++++
.../DelegationTokenAuthenticationHandler.java | 369 +++++
.../web/DelegationTokenAuthenticator.java | 261 ++++
.../web/DelegationTokenIdentifier.java | 64 +
.../delegation/web/DelegationTokenManager.java | 153 ++
.../web/HttpUserGroupInformation.java | 43 +
...rosDelegationTokenAuthenticationHandler.java | 54 +
.../KerberosDelegationTokenAuthenticator.java | 46 +
...udoDelegationTokenAuthenticationHandler.java | 55 +
.../web/PseudoDelegationTokenAuthenticator.java | 54 +
.../token/delegation/web/ServletUtils.java | 59 +
.../apache/hadoop/tracing/SpanReceiverHost.java | 153 ++
.../hadoop/util/ApplicationClassLoader.java | 219 +++
.../org/apache/hadoop/util/CacheableIPList.java | 76 +
.../java/org/apache/hadoop/util/Classpath.java | 125 ++
.../apache/hadoop/util/CombinedIPWhiteList.java | 60 +
.../org/apache/hadoop/util/DataChecksum.java | 22 +-
.../org/apache/hadoop/util/DiskChecker.java | 14 +
.../org/apache/hadoop/util/FileBasedIPList.java | 102 ++
.../hadoop/util/GenericOptionsParser.java | 6 +
.../java/org/apache/hadoop/util/IPList.java | 33 +
.../org/apache/hadoop/util/MachineList.java | 13 +-
.../apache/hadoop/util/NativeCodeLoader.java | 5 +
.../org/apache/hadoop/util/NativeCrc32.java | 43 +-
.../hadoop/util/NativeLibraryChecker.java | 21 +-
.../apache/hadoop/util/PerformanceAdvisory.java | 25 +
.../java/org/apache/hadoop/util/ProtoUtil.java | 11 +
.../java/org/apache/hadoop/util/RunJar.java | 115 +-
.../org/apache/hadoop/util/StringUtils.java | 14 +
.../java/org/apache/hadoop/util/ToolRunner.java | 2 +-
.../org/apache/hadoop/util/VersionInfo.java | 3 +-
.../hadoop-common/src/main/native/README | 7 +-
.../org/apache/hadoop/crypto/OpensslCipher.c | 382 +++++
.../hadoop/crypto/org_apache_hadoop_crypto.h | 61 +
.../hadoop/crypto/random/OpensslSecureRandom.c | 335 +++++
.../random/org_apache_hadoop_crypto_random.h | 40 +
.../org/apache/hadoop/io/nativeio/NativeIO.c | 72 +-
.../apache/hadoop/security/hadoop_user_info.c | 10 +
.../org/apache/hadoop/util/NativeCodeLoader.c | 10 +
.../src/org/apache/hadoop/util/NativeCrc32.c | 96 +-
.../src/org/apache/hadoop/util/bulk_crc32.c | 60 +-
.../src/org/apache/hadoop/util/bulk_crc32.h | 49 +-
.../org/apache/hadoop/util/test_bulk_crc32.c | 6 +-
.../src/main/proto/RpcHeader.proto | 13 +
.../src/main/resources/core-default.xml | 88 +-
.../src/site/apt/CommandsManual.apt.vm | 380 +----
.../src/site/apt/Compatibility.apt.vm | 9 +-
.../src/site/apt/FileSystemShell.apt.vm | 17 +-
.../src/site/apt/NativeLibraries.apt.vm | 22 +-
.../src/site/apt/ServiceLevelAuth.apt.vm | 21 +
.../hadoop-common/src/site/apt/Tracing.apt.vm | 169 +++
.../apache/hadoop/conf/TestConfiguration.java | 8 +
.../hadoop/crypto/CryptoStreamsTestBase.java | 721 +++++++++
.../apache/hadoop/crypto/TestCryptoCodec.java | 186 +++
.../apache/hadoop/crypto/TestCryptoStreams.java | 376 +++++
.../crypto/TestCryptoStreamsForLocalFS.java | 120 ++
.../hadoop/crypto/TestCryptoStreamsNormal.java | 123 ++
...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 31 +
.../apache/hadoop/crypto/TestOpensslCipher.java | 110 ++
.../crypto/key/TestCachingKeyProvider.java | 6 +
.../hadoop/crypto/key/TestKeyProvider.java | 17 +-
.../key/TestKeyProviderCryptoExtension.java | 11 +-
...TestKeyProviderDelegationTokenExtension.java | 13 +-
.../crypto/key/TestKeyProviderFactory.java | 79 +-
.../apache/hadoop/crypto/key/TestKeyShell.java | 56 +-
.../crypto/random/TestOpensslSecureRandom.java | 114 ++
.../crypto/random/TestOsSecureRandom.java | 139 ++
.../apache/hadoop/fs/TestContentSummary.java | 248 ++++
.../org/apache/hadoop/fs/TestHarFileSystem.java | 3 +
.../org/apache/hadoop/fs/shell/TestCount.java | 270 ++++
.../apache/hadoop/fs/shell/TestTextCommand.java | 77 +-
.../org/apache/hadoop/http/TestHttpServer.java | 6 +-
.../hadoop/ipc/TestDecayRpcScheduler.java | 225 +++
.../apache/hadoop/ipc/TestFairCallQueue.java | 392 +++++
.../java/org/apache/hadoop/ipc/TestRPC.java | 8 +-
.../hadoop/security/TestLdapGroupsMapping.java | 58 +
.../hadoop/security/TestNetgroupCache.java | 127 ++
.../security/TestWhitelistBasedResolver.java | 163 ++
.../hadoop/security/alias/TestCredShell.java | 49 +-
.../authorize/TestServiceAuthorization.java | 117 ++
.../hadoop/security/ssl/KeyStoreTestUtil.java | 41 +
.../hadoop/security/ssl/TestSSLFactory.java | 58 +-
...tionTokenAuthenticationHandlerWithMocks.java | 364 +++++
.../web/TestDelegationTokenManager.java | 59 +
.../delegation/web/TestWebDelegationToken.java | 913 ++++++++++++
.../apache/hadoop/util/ClassLoaderCheck.java | 33 +
.../hadoop/util/ClassLoaderCheckMain.java | 34 +
.../hadoop/util/ClassLoaderCheckSecond.java | 24 +
.../hadoop/util/ClassLoaderCheckThird.java | 24 +
.../hadoop/util/TestApplicationClassLoader.java | 136 ++
.../apache/hadoop/util/TestCacheableIPList.java | 188 +++
.../org/apache/hadoop/util/TestClasspath.java | 176 +++
.../apache/hadoop/util/TestDataChecksum.java | 154 +-
.../apache/hadoop/util/TestFileBasedIPList.java | 215 +++
.../hadoop/util/TestGenericOptionsParser.java | 66 +
.../hadoop/util/TestNativeCodeLoader.java | 4 +
.../java/org/apache/hadoop/util/TestRunJar.java | 66 +-
.../src/test/resources/testConf.xml | 24 +-
hadoop-common-project/hadoop-kms/pom.xml | 1 -
.../hadoop/crypto/key/kms/server/KMS.java | 356 +++--
.../hadoop/crypto/key/kms/server/KMSACLs.java | 3 +-
.../hadoop/crypto/key/kms/server/KMSAudit.java | 192 ++-
.../key/kms/server/KMSAuthenticationFilter.java | 38 +-
.../crypto/key/kms/server/KMSConfiguration.java | 9 +-
.../key/kms/server/KMSExceptionsProvider.java | 16 +-
.../crypto/key/kms/server/KMSJMXServlet.java | 36 +
.../crypto/key/kms/server/KMSMDCFilter.java | 17 +-
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 20 +-
.../hadoop-kms/src/main/webapp/WEB-INF/web.xml | 2 +-
.../hadoop-kms/src/site/apt/index.apt.vm | 99 ++
.../hadoop/crypto/key/kms/server/TestKMS.java | 261 +++-
.../crypto/key/kms/server/TestKMSACLs.java | 10 +-
.../crypto/key/kms/server/TestKMSAudit.java | 135 ++
.../test/resources/log4j-kmsaudit.properties | 25 +
.../hadoop/nfs/nfs3/Nfs3FileAttributes.java | 14 +-
.../hadoop/nfs/nfs3/request/ACCESS3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/COMMIT3Request.java | 23 +-
.../hadoop/nfs/nfs3/request/CREATE3Request.java | 18 +-
.../hadoop/nfs/nfs3/request/FSINFO3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/FSSTAT3Request.java | 15 +-
.../nfs/nfs3/request/GETATTR3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/LINK3Request.java | 61 +
.../hadoop/nfs/nfs3/request/LOOKUP3Request.java | 9 +-
.../hadoop/nfs/nfs3/request/MKDIR3Request.java | 26 +-
.../hadoop/nfs/nfs3/request/MKNOD3Request.java | 89 ++
.../hadoop/nfs/nfs3/request/NFS3Request.java | 45 +
.../nfs/nfs3/request/PATHCONF3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/READ3Request.java | 9 +-
.../nfs/nfs3/request/READDIR3Request.java | 30 +-
.../nfs/nfs3/request/READDIRPLUS3Request.java | 33 +-
.../nfs/nfs3/request/READLINK3Request.java | 15 +-
.../hadoop/nfs/nfs3/request/REMOVE3Request.java | 20 +-
.../hadoop/nfs/nfs3/request/RENAME3Request.java | 37 +-
.../hadoop/nfs/nfs3/request/RMDIR3Request.java | 20 +-
.../nfs/nfs3/request/RequestWithHandle.java | 16 +-
.../nfs/nfs3/request/SETATTR3Request.java | 29 +-
.../nfs/nfs3/request/SYMLINK3Request.java | 30 +-
.../hadoop/nfs/nfs3/request/SetAttr3.java | 9 +
.../hadoop/nfs/nfs3/request/WRITE3Request.java | 13 +-
.../hadoop/nfs/nfs3/response/LINK3Response.java | 54 +
.../nfs/nfs3/response/MKNOD3Response.java | 68 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 1 -
.../hadoop/fs/http/client/HttpFSFileSystem.java | 127 +-
.../client/HttpFSKerberosAuthenticator.java | 188 ---
.../http/client/HttpFSPseudoAuthenticator.java | 47 -
.../http/server/HttpFSAuthenticationFilter.java | 25 +-
.../HttpFSKerberosAuthenticationHandler.java | 230 ---
.../http/server/HttpFSParametersProvider.java | 108 +-
.../hadoop/fs/http/server/HttpFSServer.java | 144 +-
.../fs/http/server/HttpFSServerWebApp.java | 4 -
.../lib/service/DelegationTokenIdentifier.java | 61 -
.../lib/service/DelegationTokenManager.java | 78 -
.../DelegationTokenManagerException.java | 51 -
.../apache/hadoop/lib/service/ProxyUser.java | 31 -
.../security/DelegationTokenManagerService.java | 242 ---
.../lib/service/security/ProxyUserService.java | 179 ---
.../apache/hadoop/lib/wsrs/UserProvider.java | 109 --
.../src/main/resources/httpfs-default.xml | 15 +-
...KerberosAuthenticationHandlerForTesting.java | 6 +-
.../http/server/TestHttpFSCustomUserName.java | 94 --
...TestHttpFSKerberosAuthenticationHandler.java | 316 ----
.../hadoop/fs/http/server/TestHttpFSServer.java | 12 +-
.../fs/http/server/TestHttpFSWithKerberos.java | 6 +-
.../TestDelegationTokenManagerService.java | 89 --
.../service/security/TestProxyUserService.java | 226 ---
.../hadoop/lib/wsrs/TestUserProvider.java | 142 --
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 507 ++++---
.../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 593 ++++++++
.../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 77 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 385 ++++-
.../dev-support/findbugsExcludeFile.xml | 9 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 128 +-
.../hadoop-hdfs/src/CMakeLists.txt | 78 +-
.../bkjournal/BookKeeperEditLogInputStream.java | 5 +
.../src/main/bin/distribute-exclude.sh | 6 +-
.../hadoop-hdfs/src/main/bin/hdfs | 434 +++---
.../hadoop-hdfs/src/main/bin/hdfs-config.sh | 75 +-
.../src/main/bin/refresh-namenodes.sh | 40 +-
.../hadoop-hdfs/src/main/bin/start-balancer.sh | 30 +-
.../hadoop-hdfs/src/main/bin/start-dfs.sh | 142 +-
.../src/main/bin/start-secure-dns.sh | 34 +-
.../hadoop-hdfs/src/main/bin/stop-balancer.sh | 30 +-
.../hadoop-hdfs/src/main/bin/stop-dfs.sh | 105 +-
.../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 34 +-
.../src/main/conf/hadoop-metrics2.properties | 44 -
.../main/java/org/apache/hadoop/fs/Hdfs.java | 26 +-
.../main/java/org/apache/hadoop/fs/XAttr.java | 13 +-
.../apache/hadoop/hdfs/BlockReaderFactory.java | 27 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 207 ++-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 21 +-
.../hadoop/hdfs/DFSInotifyEventInputStream.java | 220 +++
.../org/apache/hadoop/hdfs/DFSInputStream.java | 8 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 92 +-
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 77 +-
.../hadoop/hdfs/DistributedFileSystem.java | 81 +-
.../org/apache/hadoop/hdfs/StorageType.java | 10 +
.../hdfs/UnknownCipherSuiteException.java | 38 +
.../org/apache/hadoop/hdfs/XAttrHelper.java | 8 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 100 ++
.../hadoop/hdfs/client/HdfsDataInputStream.java | 38 +-
.../hdfs/client/HdfsDataOutputStream.java | 36 +-
.../org/apache/hadoop/hdfs/inotify/Event.java | 452 ++++++
.../apache/hadoop/hdfs/inotify/EventsList.java | 63 +
.../hdfs/inotify/MissingEventsException.java | 54 +
.../org/apache/hadoop/hdfs/protocol/Block.java | 14 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 131 +-
.../hadoop/hdfs/protocol/EncryptionZone.java | 79 +
.../hdfs/protocol/EncryptionZoneIterator.java | 51 +
.../hdfs/protocol/EncryptionZoneWithId.java | 81 +
.../protocol/EncryptionZoneWithIdIterator.java | 53 +
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 53 +-
.../hdfs/protocol/HdfsLocatedFileStatus.java | 10 +-
.../hadoop/hdfs/protocol/LocatedBlocks.java | 23 +-
.../protocol/SnapshottableDirectoryStatus.java | 2 +-
.../datatransfer/ReplaceDatanodeOnFailure.java | 163 +-
.../WhitelistBasedTrustedChannelResolver.java | 119 ++
.../datatransfer/sasl/DataTransferSaslUtil.java | 4 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 116 +-
.../ClientNamenodeProtocolTranslatorPB.java | 135 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 14 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 414 +++++-
.../hdfs/qjournal/client/IPCLoggerChannel.java | 78 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 3 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 1392 ++++--------------
.../hdfs/server/balancer/BalancingPolicy.java | 84 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 1029 +++++++++++++
.../hadoop/hdfs/server/balancer/ExitStatus.java | 44 +
.../hadoop/hdfs/server/balancer/KeyManager.java | 173 +++
.../hadoop/hdfs/server/balancer/Matcher.java | 51 +
.../hdfs/server/balancer/MovedBlocks.java | 124 ++
.../hdfs/server/balancer/NameNodeConnector.java | 215 +--
.../hdfs/server/blockmanagement/BlockInfo.java | 35 +-
.../BlockInfoUnderConstruction.java | 12 +-
.../server/blockmanagement/BlockManager.java | 194 ++-
.../hdfs/server/blockmanagement/BlocksMap.java | 13 +-
.../CacheReplicationMonitor.java | 55 +-
.../blockmanagement/CorruptReplicasMap.java | 19 +-
.../blockmanagement/DatanodeDescriptor.java | 48 +-
.../server/blockmanagement/DatanodeManager.java | 21 +-
.../blockmanagement/DatanodeStorageInfo.java | 30 +-
.../blockmanagement/HeartbeatManager.java | 8 +
.../PendingDataNodeMessages.java | 34 +-
.../hdfs/server/common/HdfsServerConstants.java | 5 +
.../hdfs/server/datanode/BPOfferService.java | 305 ++--
.../hdfs/server/datanode/BPServiceActor.java | 15 +-
.../server/datanode/BlockPoolSliceStorage.java | 56 +-
.../hdfs/server/datanode/BlockReceiver.java | 59 +-
.../hdfs/server/datanode/BlockSender.java | 6 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 51 +-
.../server/datanode/DataNodeLayoutVersion.java | 5 +-
.../hdfs/server/datanode/DataStorage.java | 352 ++++-
.../hdfs/server/datanode/DataXceiver.java | 26 +-
.../hdfs/server/datanode/DataXceiverServer.java | 40 +-
.../hdfs/server/datanode/DatanodeUtil.java | 36 +
.../hdfs/server/datanode/ReplicaInfo.java | 62 +-
.../server/datanode/ShortCircuitRegistry.java | 29 +-
.../hdfs/server/datanode/StorageLocation.java | 2 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 9 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 98 +-
.../impl/FsDatasetAsyncDiskService.java | 87 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 162 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 56 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 41 +-
.../server/datanode/fsdataset/impl/LDir.java | 228 ---
.../namenode/EditLogBackupInputStream.java | 5 +
.../server/namenode/EditLogFileInputStream.java | 5 +
.../server/namenode/EditLogInputStream.java | 6 +
.../namenode/EncryptionFaultInjector.java | 39 +
.../server/namenode/EncryptionZoneManager.java | 313 ++++
.../hdfs/server/namenode/FSDirectory.java | 324 +++-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 67 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../hdfs/server/namenode/FSImageFormat.java | 11 +
.../hdfs/server/namenode/FSNamesystem.java | 769 ++++++++--
.../server/namenode/FileJournalManager.java | 47 +-
.../hadoop/hdfs/server/namenode/INode.java | 59 +-
.../hdfs/server/namenode/INodeDirectory.java | 5 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 12 +-
.../hadoop/hdfs/server/namenode/INodeMap.java | 3 +-
.../hdfs/server/namenode/INodeReference.java | 4 +-
.../hdfs/server/namenode/INodeSymlink.java | 3 +-
.../namenode/InotifyFSEditLogOpTranslator.java | 146 ++
.../hadoop/hdfs/server/namenode/JournalSet.java | 38 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 8 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 165 ++-
.../namenode/RedundantEditLogInputStream.java | 5 +
.../namenode/RetryStartFileException.java | 21 +
.../hdfs/server/namenode/TransferFsImage.java | 5 +-
.../server/namenode/XAttrPermissionFilter.java | 30 +-
.../namenode/metrics/FSNamesystemMBean.java | 7 +
.../snapshot/DirectoryWithSnapshotFeature.java | 10 +-
.../web/resources/NamenodeWebHdfsMethods.java | 26 +-
.../server/protocol/BlocksWithLocations.java | 25 +-
.../server/protocol/DatanodeStorageReport.java | 42 +
.../hdfs/server/protocol/RegisterCommand.java | 3 +
.../hadoop/hdfs/shortcircuit/DfsClientShm.java | 21 +-
.../hdfs/shortcircuit/DfsClientShmManager.java | 10 +-
.../hdfs/shortcircuit/DomainSocketFactory.java | 4 +-
.../hdfs/shortcircuit/ShortCircuitShm.java | 10 +-
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 301 ++++
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
.../offlineEditsViewer/XmlEditsVisitor.java | 4 +-
.../apache/hadoop/hdfs/util/EnumCounters.java | 9 +-
.../apache/hadoop/hdfs/util/EnumDoubles.java | 128 ++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 5 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 7 +
.../hdfs/web/resources/FsActionParam.java | 58 +
.../hadoop/hdfs/web/resources/GetOpParam.java | 4 +-
.../src/main/native/fuse-dfs/CMakeLists.txt | 4 +
.../src/main/native/libhdfs/common/htable.c | 271 ++++
.../src/main/native/libhdfs/common/htable.h | 161 ++
.../src/main/native/libhdfs/exception.c | 65 +-
.../src/main/native/libhdfs/exception.h | 10 +-
.../src/main/native/libhdfs/expect.c | 8 +-
.../src/main/native/libhdfs/expect.h | 12 +
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 416 +++---
.../src/main/native/libhdfs/jni_helper.c | 265 ++--
.../src/main/native/libhdfs/jni_helper.h | 2 -
.../src/main/native/libhdfs/native_mini_dfs.c | 6 +-
.../src/main/native/libhdfs/os/mutexes.h | 55 +
.../src/main/native/libhdfs/os/posix/mutexes.c | 43 +
.../src/main/native/libhdfs/os/posix/platform.h | 34 +
.../src/main/native/libhdfs/os/posix/thread.c | 52 +
.../libhdfs/os/posix/thread_local_storage.c | 80 +
.../src/main/native/libhdfs/os/thread.h | 54 +
.../native/libhdfs/os/thread_local_storage.h | 75 +
.../main/native/libhdfs/os/windows/inttypes.h | 28 +
.../main/native/libhdfs/os/windows/mutexes.c | 52 +
.../main/native/libhdfs/os/windows/platform.h | 86 ++
.../src/main/native/libhdfs/os/windows/thread.c | 66 +
.../libhdfs/os/windows/thread_local_storage.c | 164 +++
.../src/main/native/libhdfs/os/windows/unistd.h | 29 +
.../main/native/libhdfs/test/test_libhdfs_ops.c | 232 +--
.../native/libhdfs/test/test_libhdfs_read.c | 19 +-
.../native/libhdfs/test/test_libhdfs_write.c | 29 +-
.../native/libhdfs/test/test_libhdfs_zerocopy.c | 31 +-
.../main/native/libhdfs/test_libhdfs_threaded.c | 39 +-
.../main/native/libhdfs/test_native_mini_dfs.c | 2 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 53 +
.../src/main/proto/DatanodeProtocol.proto | 24 -
.../src/main/proto/datatransfer.proto | 1 +
.../hadoop-hdfs/src/main/proto/encryption.proto | 65 +
.../hadoop-hdfs/src/main/proto/hdfs.proto | 47 +-
.../hadoop-hdfs/src/main/proto/inotify.proto | 117 ++
.../hadoop-hdfs/src/main/proto/xattr.proto | 1 +
.../src/main/resources/hdfs-default.xml | 56 +-
.../src/main/webapps/hdfs/dfshealth.html | 5 +-
.../src/main/webapps/hdfs/dfshealth.js | 11 +-
.../src/main/webapps/hdfs/index.html | 13 +-
.../src/main/webapps/secondary/index.html | 11 -
.../src/site/apt/ExtendedAttributes.apt.vm | 4 +-
.../src/site/apt/HDFSCommands.apt.vm | 408 +++++
.../src/site/apt/HdfsNfsGateway.apt.vm | 73 +-
.../src/site/apt/HdfsUserGuide.apt.vm | 29 +-
.../hadoop-hdfs/src/site/apt/LibHdfs.apt.vm | 29 +-
.../src/site/apt/TransparentEncryption.apt.vm | 206 +++
.../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 44 +
.../src/site/xdoc/HdfsRollingUpgrade.xml | 2 +-
.../org/apache/hadoop/TestGenericRefresh.java | 29 +-
.../org/apache/hadoop/TestRefreshCallQueue.java | 44 +-
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 169 +++
.../hadoop/cli/util/CLICommandCryptoAdmin.java | 21 +
.../hadoop/cli/util/CryptoAdminCmdExecutor.java | 37 +
.../org/apache/hadoop/fs/TestGlobPaths.java | 260 ++--
.../java/org/apache/hadoop/fs/TestXAttr.java | 11 +-
.../fs/viewfs/TestViewFileSystemWithXAttrs.java | 151 ++
.../hadoop/fs/viewfs/TestViewFsWithXAttrs.java | 148 ++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 100 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 243 ++-
.../hdfs/MiniDFSClusterWithNodeGroup.java | 38 +-
.../hadoop/hdfs/TestDFSClientFailover.java | 11 +
.../hadoop/hdfs/TestDFSClientRetries.java | 11 +-
.../org/apache/hadoop/hdfs/TestDFSFinalize.java | 9 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 430 ++++++
.../apache/hadoop/hdfs/TestDFSPermission.java | 77 +
.../org/apache/hadoop/hdfs/TestDFSRename.java | 44 +
.../org/apache/hadoop/hdfs/TestDFSRollback.java | 2 +-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 177 ++-
.../hdfs/TestDFSStorageStateRecovery.java | 16 +-
.../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 12 +-
.../hadoop/hdfs/TestDFSUpgradeFromImage.java | 154 +-
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 75 +-
.../hadoop/hdfs/TestDatanodeBlockScanner.java | 15 +-
.../apache/hadoop/hdfs/TestDatanodeConfig.java | 2 +
.../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 48 +
.../apache/hadoop/hdfs/TestDatanodeReport.java | 90 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 1 -
.../apache/hadoop/hdfs/TestEncryptionZones.java | 756 ++++++++++
.../org/apache/hadoop/hdfs/TestFileAppend.java | 61 +-
.../apache/hadoop/hdfs/TestFileCorruption.java | 40 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestLease.java | 9 +-
.../hdfs/TestReplaceDatanodeOnFailure.java | 45 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 350 +++++
.../org/apache/hadoop/hdfs/TestSafeMode.java | 25 +-
.../hdfs/TestWriteBlockGetsBlockLengthHint.java | 106 ++
.../apache/hadoop/hdfs/UpgradeUtilities.java | 44 +-
.../hdfs/crypto/TestHdfsCryptoStreams.java | 91 ++
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 21 +-
.../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 6 +-
.../hdfs/qjournal/client/TestQJMWithFaults.java | 2 +-
.../client/TestQuorumJournalManager.java | 2 +-
.../security/token/block/TestBlockToken.java | 2 +
.../hdfs/server/balancer/TestBalancer.java | 567 ++++++-
.../balancer/TestBalancerWithHANameNodes.java | 8 +-
.../TestBalancerWithMultipleNameNodes.java | 8 +-
.../balancer/TestBalancerWithNodeGroup.java | 12 +-
.../blockmanagement/BlockManagerTestUtil.java | 10 +
.../server/blockmanagement/TestBlockInfo.java | 20 +-
.../blockmanagement/TestBlockManager.java | 10 +-
.../blockmanagement/TestCorruptReplicaInfo.java | 14 +-
.../blockmanagement/TestDatanodeDescriptor.java | 6 +-
.../TestPendingDataNodeMessages.java | 7 +-
.../blockmanagement/TestReplicationPolicy.java | 5 +-
.../server/datanode/SimulatedFSDataset.java | 11 +
.../TestBlockHasMultipleReplicasOnSameDN.java | 5 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 1 -
.../server/datanode/TestBlockReplacement.java | 6 +-
.../datanode/TestDataNodeRollingUpgrade.java | 48 +
.../datanode/TestDataNodeVolumeFailure.java | 24 +-
.../hdfs/server/datanode/TestDataStorage.java | 210 +++
.../server/datanode/TestDeleteBlockPool.java | 7 +-
.../hdfs/server/datanode/TestDiskError.java | 4 +-
...TestDnRespectsBlockReportSplitThreshold.java | 6 +-
.../server/datanode/TestFsDatasetCache.java | 1 -
.../hdfs/server/datanode/TestStorageReport.java | 2 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 183 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 28 +
.../hdfs/server/namenode/FSXAttrBaseTest.java | 179 ++-
.../server/namenode/NNThroughputBenchmark.java | 5 +-
.../hdfs/server/namenode/TestAddBlockRetry.java | 4 +-
.../namenode/TestBlockUnderConstruction.java | 3 +
.../server/namenode/TestCacheDirectives.java | 1 -
.../TestCommitBlockSynchronization.java | 11 +
.../namenode/TestDecommissioningStatus.java | 98 +-
.../hdfs/server/namenode/TestDeleteRace.java | 133 ++
.../hdfs/server/namenode/TestEditLog.java | 4 +
.../hdfs/server/namenode/TestFSDirectory.java | 15 +-
.../server/namenode/TestFSNamesystemMBean.java | 2 +
.../hadoop/hdfs/server/namenode/TestFsck.java | 19 +-
.../hdfs/server/namenode/TestINodeFile.java | 16 +
.../namenode/TestListCorruptFileBlocks.java | 163 +-
.../namenode/TestNNStorageRetentionManager.java | 11 +-
.../server/namenode/TestNameNodeMXBean.java | 45 +-
.../server/namenode/TestNamenodeRetryCache.java | 9 +-
.../hdfs/server/namenode/TestStartup.java | 41 +
.../namenode/ha/TestPipelinesFailover.java | 62 +-
.../namenode/ha/TestRetryCacheWithHA.java | 10 +-
.../namenode/metrics/TestNameNodeMetrics.java | 2 +-
.../namenode/snapshot/TestAclWithSnapshot.java | 9 +
.../namenode/snapshot/TestSnapshotDeletion.java | 77 +-
.../shortcircuit/TestShortCircuitCache.java | 98 ++
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 39 +-
.../hadoop/hdfs/web/TestWebHDFSForHA.java | 76 +-
.../hdfs/web/TestWebHdfsFileSystemContract.java | 33 +
.../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 24 +
.../hadoop/security/TestPermissionSymlinks.java | 35 +
.../security/TestRefreshUserMappings.java | 12 +-
.../org/apache/hadoop/tracing/TestTracing.java | 280 ++++
.../src/test/resources/hadoop-0.23-reserved.tgz | Bin 0 -> 4558 bytes
.../src/test/resources/hadoop-1-reserved.tgz | Bin 0 -> 2572 bytes
.../test/resources/hadoop-24-datanode-dir.tgz | Bin 0 -> 637608 bytes
.../src/test/resources/hadoop-datanode-dir.txt | 23 +
.../src/test/resources/testCryptoConf.xml | 284 ++++
.../src/test/resources/testHDFSConf.xml | 44 +
.../src/test/resources/testXAttrConf.xml | 58 +-
hadoop-mapreduce-project/CHANGES.txt | 109 +-
hadoop-mapreduce-project/bin/mapred | 226 +--
hadoop-mapreduce-project/bin/mapred-config.sh | 76 +-
.../bin/mr-jobhistory-daemon.sh | 143 +-
hadoop-mapreduce-project/conf/mapred-env.sh | 60 +-
.../hadoop-mapreduce-client-app/pom.xml | 6 +
.../hadoop/mapred/LocalContainerLauncher.java | 18 +-
.../hadoop/mapred/TaskAttemptListenerImpl.java | 4 +-
.../org/apache/hadoop/mapred/YarnChild.java | 22 +-
.../jobhistory/JobHistoryEventHandler.java | 12 +-
.../hadoop/mapreduce/v2/app/AppContext.java | 1 +
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 5 +
.../v2/app/client/MRClientService.java | 3 +-
.../v2/app/commit/CommitterEventHandler.java | 2 +-
.../mapreduce/v2/app/job/impl/JobImpl.java | 30 +-
.../mapreduce/v2/app/rm/RMCommunicator.java | 2 +-
.../v2/app/rm/RMContainerAllocator.java | 26 +-
.../v2/app/speculate/DefaultSpeculator.java | 8 +-
.../mapreduce/v2/app/webapp/TaskPage.java | 3 +-
.../mapreduce/v2/app/webapp/TasksBlock.java | 4 +-
.../mapred/TestTaskAttemptListenerImpl.java | 13 +-
.../jobhistory/TestJobHistoryEventHandler.java | 127 +-
.../hadoop/mapreduce/v2/app/MockAppContext.java | 5 +
.../mapreduce/v2/app/TestJobEndNotifier.java | 3 +-
.../mapreduce/v2/app/TestRuntimeEstimators.java | 5 +
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 10 +-
.../mapreduce/v2/app/webapp/TestBlocks.java | 3 +
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 4 +-
.../v2/jobhistory/JobHistoryUtils.java | 79 +-
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 3 +-
.../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 9 +-
.../hadoop/mapreduce/v2/util/TestMRApps.java | 5 +-
.../org/apache/hadoop/mapred/BackupStore.java | 8 +-
.../java/org/apache/hadoop/mapred/IFile.java | 21 +-
.../org/apache/hadoop/mapred/InputFormat.java | 2 +-
.../java/org/apache/hadoop/mapred/JobConf.java | 94 +-
.../java/org/apache/hadoop/mapred/MapTask.java | 70 +-
.../java/org/apache/hadoop/mapred/Merger.java | 15 +-
.../java/org/apache/hadoop/mapred/Task.java | 6 +
.../hadoop/mapred/TaskCompletionEvent.java | 4 +-
.../mapred/join/CompositeInputFormat.java | 2 +-
.../apache/hadoop/mapreduce/CryptoUtils.java | 199 +++
.../apache/hadoop/mapreduce/InputFormat.java | 2 +-
.../java/org/apache/hadoop/mapreduce/Job.java | 11 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 9 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 23 +
.../hadoop/mapreduce/TaskCompletionEvent.java | 4 +-
.../mapreduce/lib/db/OracleDBRecordReader.java | 7 +-
.../lib/input/KeyValueTextInputFormat.java | 3 +
.../lib/join/CompositeInputFormat.java | 2 +-
.../hadoop/mapreduce/task/reduce/Fetcher.java | 14 +-
.../mapreduce/task/reduce/LocalFetcher.java | 4 +
.../mapreduce/task/reduce/MergeManagerImpl.java | 37 +-
.../mapreduce/task/reduce/OnDiskMapOutput.java | 3 +-
.../task/reduce/ShuffleSchedulerImpl.java | 34 +-
.../hadoop/mapreduce/util/ConfigUtil.java | 2 +
.../src/main/resources/mapred-default.xml | 107 +-
.../src/site/apt/MapredCommands.apt.vm | 230 +++
.../apt/PluggableShuffleAndPluggableSort.apt.vm | 8 +-
.../src/site/markdown/DistCp.md.vm | 28 +-
.../src/site/markdown/HadoopArchives.md.vm | 27 +-
.../org/apache/hadoop/mapred/TestJobConf.java | 15 +-
.../hadoop/mapreduce/lib/db/TestDbClasses.java | 2 +-
.../mapreduce/task/reduce/TestMerger.java | 146 +-
.../mapreduce/v2/hs/HistoryClientService.java | 11 +-
...istoryServerFileSystemStateStoreService.java | 101 +-
.../hadoop/mapreduce/v2/hs/JobHistory.java | 5 +
.../mapreduce/v2/hs/server/HSAdminServer.java | 5 +-
.../mapreduce/v2/hs/webapp/HsTaskPage.java | 3 +-
.../mapreduce/v2/hs/TestHistoryFileManager.java | 70 +-
...istoryServerFileSystemStateStoreService.java | 85 +-
.../mapreduce/v2/hs/webapp/TestBlocks.java | 4 +-
.../org/apache/hadoop/mapred/YARNRunner.java | 9 +
.../test/java/org/apache/hadoop/SleepJob.java | 275 ----
.../org/apache/hadoop/conf/TestJobConf.java | 20 +-
.../org/apache/hadoop/fs/JHLogAnalyzer.java | 4 +-
.../apache/hadoop/mapred/TestComparators.java | 61 +-
.../org/apache/hadoop/mapred/TestIFile.java | 11 +-
.../TestMRIntermediateDataEncryption.java | 254 ++++
.../apache/hadoop/mapred/TestMapOutputType.java | 62 +-
.../org/apache/hadoop/mapred/TestMapRed.java | 28 +-
.../apache/hadoop/mapred/TestReduceTask.java | 2 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 104 +-
.../mapred/lib/TestKeyFieldBasedComparator.java | 25 +-
.../mapred/pipes/TestPipeApplication.java | 12 +-
.../org/apache/hadoop/mapreduce/SleepJob.java | 32 +-
.../apache/hadoop/mapreduce/TestMapReduce.java | 30 +-
.../TestMRAMWithNonNormalizedCapabilities.java | 2 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 9 +-
.../v2/TestMRJobsWithHistoryService.java | 2 +-
.../mapreduce/v2/TestMRJobsWithProfiler.java | 2 +-
.../apache/hadoop/mapred/ShuffleHandler.java | 28 +-
.../hadoop/mapred/TestShuffleHandler.java | 8 +-
hadoop-project-dist/pom.xml | 14 +
hadoop-project/pom.xml | 49 +-
hadoop-project/src/site/site.xml | 5 +
.../org/apache/hadoop/tools/HadoopArchives.java | 14 +-
.../apache/hadoop/tools/TestHadoopArchives.java | 41 +
.../hadoop-aws/dev-support/findbugs-exclude.xml | 19 +
hadoop-tools/hadoop-aws/pom.xml | 116 ++
.../apache/hadoop/tools/DistCpConstants.java | 6 +
.../apache/hadoop/tools/DistCpOptionSwitch.java | 6 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 21 +-
.../apache/hadoop/tools/SimpleCopyListing.java | 48 +-
.../hadoop/tools/mapred/CopyCommitter.java | 9 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 10 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 48 +-
.../hadoop/tools/TestDistCpWithRawXAttrs.java | 170 +++
.../hadoop/tools/TestDistCpWithXAttrs.java | 71 +-
.../apache/hadoop/tools/TestOptionsParser.java | 3 +-
.../hadoop/tools/util/DistCpTestUtils.java | 89 ++
.../hadoop/tools/util/TestDistCpUtils.java | 6 +-
.../mapred/gridmix/TestGridmixRecord.java | 10 +-
.../hadoop/mapred/gridmix/TestHighRamJob.java | 12 +-
hadoop-tools/hadoop-openstack/pom.xml | 5 +
.../hadoop/yarn/sls/RumenToSLSConverter.java | 4 +
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 4 +
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 73 +-
.../yarn/sls/appmaster/MRAMSimulator.java | 9 +-
.../hadoop/yarn/sls/conf/SLSConfiguration.java | 5 +
.../yarn/sls/nodemanager/NMSimulator.java | 72 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 35 +-
.../sls/scheduler/CapacitySchedulerMetrics.java | 5 +
.../yarn/sls/scheduler/ContainerSimulator.java | 4 +
.../sls/scheduler/FairSchedulerMetrics.java | 12 +-
.../sls/scheduler/FifoSchedulerMetrics.java | 4 +
.../NodeUpdateSchedulerEventWrapper.java | 4 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 15 +-
.../sls/scheduler/ResourceSchedulerWrapper.java | 50 +-
.../sls/scheduler/SLSCapacityScheduler.java | 26 +-
.../yarn/sls/scheduler/SchedulerMetrics.java | 4 +
.../yarn/sls/scheduler/SchedulerWrapper.java | 4 +
.../hadoop/yarn/sls/scheduler/TaskRunner.java | 22 +-
.../apache/hadoop/yarn/sls/utils/SLSUtils.java | 4 +
.../apache/hadoop/yarn/sls/web/SLSWebApp.java | 4 +
.../apache/hadoop/yarn/sls/TestSLSRunner.java | 30 +-
.../yarn/sls/appmaster/TestAMSimulator.java | 86 ++
.../yarn/sls/nodemanager/TestNMSimulator.java | 100 ++
hadoop-tools/pom.xml | 1 +
hadoop-yarn-project/CHANGES.txt | 194 ++-
hadoop-yarn-project/hadoop-yarn/bin/slaves.sh | 70 -
.../hadoop-yarn/bin/start-yarn.sh | 34 +-
.../hadoop-yarn/bin/stop-yarn.sh | 40 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 398 ++---
.../hadoop-yarn/bin/yarn-config.sh | 120 +-
.../hadoop-yarn/bin/yarn-daemon.sh | 157 +-
.../hadoop-yarn/bin/yarn-daemons.sh | 38 +-
.../hadoop-yarn/conf/yarn-env.sh | 163 +-
.../dev-support/findbugs-exclude.xml | 14 +
.../api/protocolrecords/AllocateResponse.java | 32 +
.../protocolrecords/GetApplicationsRequest.java | 18 +
.../hadoop/yarn/api/records/ContainerId.java | 10 +-
.../hadoop/yarn/api/records/ResourceOption.java | 4 +
.../hadoop/yarn/conf/YarnConfiguration.java | 40 +-
.../UpdateNodeResourceResponse.java | 12 +-
..._server_resourcemanager_service_protos.proto | 63 -
.../src/main/proto/yarn_service_protos.proto | 1 +
.../distributedshell/ApplicationMaster.java | 31 +-
.../applications/distributedshell/Client.java | 2 +-
.../distributedshell/TestDSFailedAppMaster.java | 6 +-
.../distributedshell/TestDistributedShell.java | 75 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 8 -
.../hadoop/yarn/client/api/AMRMClient.java | 63 +
.../hadoop/yarn/client/api/TimelineClient.java | 88 --
.../yarn/client/api/async/AMRMClientAsync.java | 64 +
.../yarn/client/api/impl/AMRMClientImpl.java | 19 +
.../client/api/impl/TimelineAuthenticator.java | 260 ----
.../client/api/impl/TimelineClientImpl.java | 332 -----
.../hadoop/yarn/client/ProtocolHATestBase.java | 1 +
.../TestApplicationMasterServiceOnHA.java | 6 +-
.../api/async/impl/TestAMRMClientAsync.java | 76 +-
.../yarn/client/api/impl/TestAMRMClient.java | 170 ++-
.../api/impl/TestAMRMClientOnRMRestart.java | 151 ++
.../api/impl/TestTimelineAuthenticator.java | 40 -
.../client/api/impl/TestTimelineClient.java | 206 ---
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 8 +
.../impl/pb/AllocateResponsePBImpl.java | 38 +-
.../impl/pb/GetApplicationsRequestPBImpl.java | 10 +
.../impl/pb/ApplicationReportPBImpl.java | 1 +
.../pb/ApplicationSubmissionContextPBImpl.java | 1 +
.../impl/pb/ResourceBlacklistRequestPBImpl.java | 13 +-
.../records/impl/pb/ResourceOptionPBImpl.java | 48 +-
.../yarn/api/records/impl/pb/TokenPBImpl.java | 2 +-
.../hadoop/yarn/client/api/TimelineClient.java | 88 ++
.../client/api/impl/TimelineAuthenticator.java | 269 ++++
.../client/api/impl/TimelineClientImpl.java | 406 +++++
.../yarn/client/api/impl/package-info.java | 21 +
.../hadoop/yarn/client/api/package-info.java | 21 +
.../pb/UpdateNodeResourceRequestPBImpl.java | 16 +-
.../pb/UpdateNodeResourceResponsePBImpl.java | 2 +-
.../yarn/util/ApplicationClassLoader.java | 170 +--
.../resource/DefaultResourceCalculator.java | 7 +
.../resource/DominantResourceCalculator.java | 8 +
.../yarn/util/resource/ResourceCalculator.java | 9 +
.../hadoop/yarn/util/resource/Resources.java | 5 +
.../hadoop/yarn/webapp/util/WebAppUtils.java | 83 +-
.../src/main/resources/yarn-default.xml | 68 +-
.../apache/hadoop/yarn/api/TestContainerId.java | 4 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 895 +++++++++++
.../api/impl/TestTimelineAuthenticator.java | 40 +
.../client/api/impl/TestTimelineClient.java | 206 +++
.../hadoop/yarn/conf/TestYarnConfiguration.java | 128 ++
.../yarn/util/TestApplicationClassLoader.java | 136 --
.../yarn/webapp/util/TestWebAppUtils.java | 148 ++
.../ApplicationHistoryClientService.java | 16 +-
.../ApplicationHistoryServer.java | 117 +-
.../FileSystemApplicationHistoryStore.java | 14 +-
.../webapp/AHSWebApp.java | 25 +-
.../server/timeline/LeveldbTimelineStore.java | 74 +-
.../server/timeline/TimelineDataManager.java | 319 ++++
.../timeline/webapp/CrossOriginFilter.java | 220 +++
.../webapp/CrossOriginFilterInitializer.java | 42 +
.../timeline/webapp/TimelineWebServices.java | 255 +---
.../TestApplicationHistoryClientService.java | 2 +-
.../TestApplicationHistoryServer.java | 41 +
.../TestFileSystemApplicationHistoryStore.java | 62 +-
.../timeline/TestLeveldbTimelineStore.java | 53 +-
.../timeline/webapp/TestCrossOriginFilter.java | 214 +++
.../TestCrossOriginFilterInitializer.java | 60 +
.../webapp/TestTimelineWebServices.java | 9 +-
.../webapp/TestTimelineWebServicesWithSSL.java | 134 ++
.../hadoop/yarn/server/records/Version.java | 84 ++
.../server/records/impl/pb/VersionPBImpl.java | 77 +
.../security/http/RMAuthenticationFilter.java | 63 +
.../http/RMAuthenticationFilterInitializer.java | 121 ++
.../main/proto/yarn_server_common_protos.proto | 8 +-
.../server/nodemanager/ContainerExecutor.java | 73 +-
.../nodemanager/DefaultContainerExecutor.java | 59 +-
.../nodemanager/LinuxContainerExecutor.java | 25 +-
.../server/nodemanager/NodeStatusUpdater.java | 23 +
.../nodemanager/NodeStatusUpdaterImpl.java | 34 +-
.../containermanager/ContainerManagerImpl.java | 260 +++-
.../application/ApplicationImpl.java | 6 +
.../container/ContainerImpl.java | 165 ++-
.../launcher/ContainerLaunch.java | 46 +-
.../launcher/ContainersLauncher.java | 15 +-
.../launcher/ContainersLauncherEventType.java | 1 +
.../launcher/RecoveredContainerLaunch.java | 126 ++
.../localizer/ResourceLocalizationService.java | 10 +-
.../logaggregation/AppLogAggregator.java | 2 +
.../logaggregation/AppLogAggregatorImpl.java | 14 +-
.../logaggregation/LogAggregationService.java | 10 +-
.../recovery/NMLeveldbStateStoreService.java | 287 +++-
.../recovery/NMNullStateStoreService.java | 58 +
.../recovery/NMStateStoreService.java | 212 +++
.../recovery/records/NMDBSchemaVersion.java | 80 -
.../impl/pb/NMDBSchemaVersionPBImpl.java | 81 -
.../server/nodemanager/webapp/WebServer.java | 4 +-
.../impl/container-executor.c | 111 +-
.../yarn_server_nodemanager_recovery.proto | 12 +-
.../nodemanager/TestLinuxContainerExecutor.java | 7 +
.../nodemanager/TestNodeManagerResync.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 17 +-
.../BaseContainerManagerTest.java | 2 +-
.../containermanager/TestAuxServices.java | 3 +-
.../TestContainerManagerRecovery.java | 324 ++++
.../container/TestContainer.java | 6 +-
.../recovery/NMMemoryStateStoreService.java | 152 +-
.../TestNMLeveldbStateStoreService.java | 187 ++-
.../nodemanager/webapp/TestNMWebServer.java | 2 +-
.../conf/capacity-scheduler.xml | 23 +
.../hadoop-yarn-server-resourcemanager/pom.xml | 31 +
.../server/resourcemanager/AdminService.java | 50 +-
.../ApplicationMasterService.java | 41 +-
.../server/resourcemanager/ClientRMService.java | 8 +-
.../resourcemanager/EmbeddedElectorService.java | 10 +-
.../resourcemanager/RMSecretManagerService.java | 6 +-
.../server/resourcemanager/ResourceManager.java | 103 +-
.../resourcemanager/ResourceTrackerService.java | 14 +-
.../resourcemanager/amlauncher/AMLauncher.java | 11 +-
.../recovery/FileSystemRMStateStore.java | 87 +-
.../recovery/MemoryRMStateStore.java | 21 +-
.../recovery/NullRMStateStore.java | 13 +-
.../resourcemanager/recovery/RMStateStore.java | 108 +-
.../recovery/RMStateStoreFactory.java | 13 +-
.../recovery/ZKRMStateStore.java | 78 +-
.../records/AMRMTokenSecretManagerState.java | 76 +
.../records/ApplicationAttemptStateData.java | 2 +-
.../recovery/records/ApplicationStateData.java | 2 +-
.../resourcemanager/recovery/records/Epoch.java | 2 +-
.../recovery/records/RMStateVersion.java | 80 -
.../pb/AMRMTokenSecretManagerStatePBImpl.java | 126 ++
.../pb/ApplicationAttemptStateDataPBImpl.java | 6 +-
.../impl/pb/ApplicationStateDataPBImpl.java | 6 +-
.../recovery/records/impl/pb/EpochPBImpl.java | 4 +-
.../records/impl/pb/RMStateVersionPBImpl.java | 76 -
.../server/resourcemanager/rmapp/RMAppImpl.java | 40 +-
.../rmapp/RMAppNewSavedEvent.java | 36 -
.../rmapp/RMAppUpdateSavedEvent.java | 36 -
.../rmapp/attempt/RMAppAttemptImpl.java | 84 +-
.../event/RMAppAttemptNewSavedEvent.java | 39 -
.../event/RMAppAttemptUpdateSavedEvent.java | 38 -
.../server/resourcemanager/rmnode/RMNode.java | 12 -
.../resourcemanager/rmnode/RMNodeEventType.java | 3 +
.../resourcemanager/rmnode/RMNodeImpl.java | 162 +-
.../rmnode/RMNodeReconnectEvent.java | 12 +-
.../rmnode/RMNodeResourceUpdateEvent.java | 37 +
.../scheduler/AbstractYarnScheduler.java | 185 +++
.../scheduler/AppSchedulingInfo.java | 5 +-
.../scheduler/SchedulerApplicationAttempt.java | 18 +-
.../scheduler/SchedulerNode.java | 25 +-
.../scheduler/SchedulerUtils.java | 32 -
.../scheduler/YarnScheduler.java | 18 +
.../scheduler/capacity/CSQueue.java | 18 +
.../scheduler/capacity/CSQueueUtils.java | 53 +
.../scheduler/capacity/CapacityScheduler.java | 223 ++-
.../CapacitySchedulerConfiguration.java | 119 +-
.../scheduler/capacity/LeafQueue.java | 50 +-
.../scheduler/capacity/ParentQueue.java | 33 +
.../event/NodeResourceUpdateSchedulerEvent.java | 43 +
.../scheduler/event/SchedulerEventType.java | 1 +
.../scheduler/fair/AllocationConfiguration.java | 40 +-
.../fair/AllocationFileLoaderService.java | 57 +-
.../scheduler/fair/AppSchedulable.java | 463 ------
.../scheduler/fair/FSAppAttempt.java | 762 ++++++++++
.../scheduler/fair/FSLeafQueue.java | 85 +-
.../scheduler/fair/FSOpDurations.java | 119 ++
.../scheduler/fair/FSParentQueue.java | 25 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 85 +-
.../scheduler/fair/FSQueueMetrics.java | 17 +-
.../scheduler/fair/FSSchedulerApp.java | 360 -----
.../scheduler/fair/FSSchedulerNode.java | 6 +-
.../scheduler/fair/FairScheduler.java | 227 +--
.../scheduler/fair/FifoAppComparator.java | 8 +-
.../scheduler/fair/MaxRunningAppsEnforcer.java | 52 +-
.../scheduler/fair/NewAppWeightBooster.java | 2 +-
.../scheduler/fair/QueueManager.java | 37 +-
.../scheduler/fair/Schedulable.java | 61 +-
.../scheduler/fair/SchedulingPolicy.java | 27 +-
.../scheduler/fair/WeightAdjuster.java | 2 +-
.../fair/policies/ComputeFairShares.java | 55 +-
.../DominantResourceFairnessPolicy.java | 9 +
.../fair/policies/FairSharePolicy.java | 8 +
.../scheduler/fair/policies/FifoPolicy.java | 8 +
.../scheduler/fifo/FifoScheduler.java | 42 +-
.../security/AMRMTokenSecretManager.java | 53 +-
.../security/DelegationTokenRenewer.java | 2 +-
.../security/RMAuthenticationHandler.java | 157 ++
.../resourcemanager/webapp/AboutBlock.java | 1 +
.../webapp/FairSchedulerAppsBlock.java | 4 +
.../webapp/FairSchedulerPage.java | 31 +-
.../resourcemanager/webapp/RMWebServices.java | 26 +-
.../resourcemanager/webapp/dao/ClusterInfo.java | 8 +
.../webapp/dao/FairSchedulerInfo.java | 16 +-
.../webapp/dao/FairSchedulerLeafQueueInfo.java | 9 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 34 +-
.../yarn_server_resourcemanager_recovery.proto | 90 ++
.../yarn/server/resourcemanager/MockAM.java | 8 +-
.../yarn/server/resourcemanager/MockNodes.java | 22 +-
.../yarn/server/resourcemanager/MockRM.java | 15 +-
.../MockRMWithCustomAMLauncher.java | 5 +-
.../resourcemanager/TestApplicationCleanup.java | 29 +
.../TestApplicationMasterService.java | 1 -
.../resourcemanager/TestFifoScheduler.java | 85 ++
.../yarn/server/resourcemanager/TestRMHA.java | 46 +-
.../resourcemanager/TestRMNodeTransitions.java | 80 +-
.../server/resourcemanager/TestRMRestart.java | 25 +-
.../resourcemanager/TestResourceManager.java | 76 +
.../TestResourceTrackerService.java | 2 +-
.../TestWorkPreservingRMRestart.java | 101 +-
.../applicationsmanager/TestAMRestart.java | 9 +-
.../recovery/RMStateStoreTestBase.java | 108 +-
.../recovery/TestFSRMStateStore.java | 14 +-
.../recovery/TestZKRMStateStore.java | 13 +-
.../resourcetracker/TestNMReconnect.java | 19 +-
.../rmapp/TestRMAppTransitions.java | 15 +-
.../attempt/TestRMAppAttemptTransitions.java | 48 +-
.../scheduler/capacity/TestCSQueueUtils.java | 262 ++++
.../capacity/TestCapacityScheduler.java | 983 ++++++++++++-
.../scheduler/capacity/TestQueueMappings.java | 240 +++
.../scheduler/capacity/TestUtils.java | 5 +-
.../scheduler/fair/FakeSchedulable.java | 13 +-
.../fair/TestAllocationFileLoaderService.java | 64 +-
.../scheduler/fair/TestFSAppAttempt.java | 188 +++
.../scheduler/fair/TestFSLeafQueue.java | 2 +-
.../scheduler/fair/TestFSSchedulerApp.java | 191 ---
.../scheduler/fair/TestFairScheduler.java | 684 ++++++++-
.../fair/TestFairSchedulerFairShare.java | 360 +++++
.../fair/TestMaxRunningAppsEnforcer.java | 42 +-
.../scheduler/fifo/TestFifoScheduler.java | 24 +-
.../security/TestAMRMTokens.java | 55 +-
.../webapp/TestRMWebAppFairScheduler.java | 96 +-
.../webapp/TestRMWebServices.java | 2 +-
.../TestRMWebServicesAppsModification.java | 21 +-
...ebServicesDelegationTokenAuthentication.java | 357 +++++
.../webapp/TestRMWebappAuthentication.java | 272 ++++
.../src/site/apt/FairScheduler.apt.vm | 38 +-
.../src/site/apt/ResourceManagerRest.apt.vm | 21 +
.../src/site/apt/WritingYarnApplications.apt.vm | 1369 +++++++++--------
977 files changed, 60546 insertions(+), 16363 deletions(-)
----------------------------------------------------------------------
