Merge remote-tracking branch 'origin/trunk' into MR-2841 Conflicts: hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cce7d1e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cce7d1e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cce7d1e2 Branch: refs/heads/HDFS-6584 Commit: cce7d1e2f96ff75fe19ab2879ddd90a898ca5b18 Parents: 7ade9b0 727331b Author: Todd Lipcon <t...@apache.org> Authored: Tue Sep 2 15:55:54 2014 -0700 Committer: Todd Lipcon <t...@apache.org> Committed: Tue Sep 2 15:55:54 2014 -0700 ---------------------------------------------------------------------- .gitignore | 1 + BUILDING.txt | 22 + dev-support/test-patch.sh | 26 +- .../main/resources/assemblies/hadoop-dist.xml | 7 + .../dev-support/findbugsExcludeFile.xml | 38 + hadoop-common-project/hadoop-auth/pom.xml | 26 + .../authentication/client/AuthenticatedURL.java | 36 +- .../server/AuthenticationFilter.java | 109 +- .../server/KerberosAuthenticationHandler.java | 21 +- .../server/PseudoAuthenticationHandler.java | 21 +- .../util/RandomSignerSecretProvider.java | 49 + .../util/RolloverSignerSecretProvider.java | 139 ++ .../security/authentication/util/Signer.java | 46 +- .../util/SignerSecretProvider.java | 62 + .../util/StringSignerSecretProvider.java | 49 + .../client/AuthenticatorTestCase.java | 137 +- .../client/TestAuthenticatedURL.java | 38 +- .../client/TestKerberosAuthenticator.java | 58 +- .../server/TestAuthenticationFilter.java | 118 +- .../util/TestRandomSignerSecretProvider.java | 63 + .../util/TestRolloverSignerSecretProvider.java | 79 + .../authentication/util/TestSigner.java | 85 +- .../util/TestStringSignerSecretProvider.java | 33 + hadoop-common-project/hadoop-common/CHANGES.txt | 502 +++++-- .../dev-support/findbugsExcludeFile.xml | 5 + hadoop-common-project/hadoop-common/pom.xml | 34 +- .../hadoop-common/src/CMakeLists.txt | 34 + .../hadoop-common/src/JNIFlags.cmake | 6 + .../hadoop-common/src/config.h.cmake | 1 + .../src/contrib/bash-tab-completion/hadoop.sh | 28 +- .../hadoop-common/src/main/bin/hadoop | 229 +-- .../src/main/bin/hadoop-config.cmd | 10 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 409 ++--- .../hadoop-common/src/main/bin/hadoop-daemon.sh | 214 +-- .../src/main/bin/hadoop-daemons.sh | 37 +- .../src/main/bin/hadoop-functions.sh | 1066 ++++++++++++++ .../src/main/bin/hadoop-layout.sh.example | 93 ++ .../hadoop-common/src/main/bin/hadoop.cmd | 33 +- .../hadoop-common/src/main/bin/rcc | 52 +- .../hadoop-common/src/main/bin/slaves.sh | 51 +- .../hadoop-common/src/main/bin/start-all.sh | 38 +- .../hadoop-common/src/main/bin/stop-all.sh | 36 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 432 +++++- .../org/apache/hadoop/conf/Configuration.java | 78 +- .../hadoop/conf/ReconfigurationServlet.java | 2 + .../apache/hadoop/crypto/AesCtrCryptoCodec.java | 67 + .../org/apache/hadoop/crypto/CipherSuite.java | 115 ++ .../org/apache/hadoop/crypto/CryptoCodec.java | 179 +++ .../apache/hadoop/crypto/CryptoInputStream.java | 680 +++++++++ .../hadoop/crypto/CryptoOutputStream.java | 280 ++++ .../apache/hadoop/crypto/CryptoStreamUtils.java | 70 + .../org/apache/hadoop/crypto/Decryptor.java | 72 + .../org/apache/hadoop/crypto/Encryptor.java | 71 + .../hadoop/crypto/JceAesCtrCryptoCodec.java | 165 +++ .../hadoop/crypto/OpensslAesCtrCryptoCodec.java | 164 +++ .../org/apache/hadoop/crypto/OpensslCipher.java | 289 ++++ .../hadoop/crypto/key/JavaKeyStoreProvider.java | 248 +++- .../apache/hadoop/crypto/key/KeyProvider.java | 32 +- .../crypto/key/KeyProviderCryptoExtension.java | 91 +- .../KeyProviderDelegationTokenExtension.java | 8 +- .../hadoop/crypto/key/KeyProviderExtension.java | 1 + .../org/apache/hadoop/crypto/key/KeyShell.java | 89 +- .../apache/hadoop/crypto/key/UserProvider.java | 5 +- .../crypto/key/kms/KMSClientProvider.java | 117 +- .../hadoop/crypto/key/kms/KMSRESTConstants.java | 2 +- .../crypto/random/OpensslSecureRandom.java | 122 ++ .../hadoop/crypto/random/OsSecureRandom.java | 115 ++ .../apache/hadoop/fs/AbstractFileSystem.java | 103 +- .../apache/hadoop/fs/ChecksumFileSystem.java | 8 +- .../java/org/apache/hadoop/fs/ChecksumFs.java | 8 +- .../hadoop/fs/CommonConfigurationKeys.java | 4 +- .../fs/CommonConfigurationKeysPublic.java | 34 +- .../org/apache/hadoop/fs/ContentSummary.java | 46 +- .../apache/hadoop/fs/FSDataOutputStream.java | 2 +- .../org/apache/hadoop/fs/FSOutputSummer.java | 107 +- .../java/org/apache/hadoop/fs/FileContext.java | 141 +- .../apache/hadoop/fs/FileEncryptionInfo.java | 102 ++ .../java/org/apache/hadoop/fs/FileSystem.java | 159 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 7 + .../java/org/apache/hadoop/fs/FilterFs.java | 8 + .../main/java/org/apache/hadoop/fs/Globber.java | 8 +- .../fs/crypto/CryptoFSDataInputStream.java | 37 + .../fs/crypto/CryptoFSDataOutputStream.java | 47 + .../hadoop/fs/shell/CommandWithDestination.java | 75 +- .../apache/hadoop/fs/shell/CopyCommands.java | 6 +- .../java/org/apache/hadoop/fs/shell/Count.java | 36 +- .../java/org/apache/hadoop/fs/shell/Delete.java | 6 +- .../org/apache/hadoop/fs/shell/Display.java | 12 +- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 8 + .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 7 + .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 10 +- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 9 + .../org/apache/hadoop/http/HttpServer2.java | 16 +- .../java/org/apache/hadoop/io/SequenceFile.java | 9 +- .../org/apache/hadoop/io/nativeio/NativeIO.java | 19 +- .../main/java/org/apache/hadoop/ipc/Client.java | 8 + .../apache/hadoop/ipc/DecayRpcScheduler.java | 522 +++++++ .../hadoop/ipc/DecayRpcSchedulerMXBean.java | 30 + .../org/apache/hadoop/ipc/FairCallQueue.java | 449 ++++++ .../apache/hadoop/ipc/FairCallQueueMXBean.java | 27 + .../apache/hadoop/ipc/ProtobufRpcEngine.java | 20 +- .../org/apache/hadoop/ipc/RpcMultiplexer.java | 32 + .../org/apache/hadoop/ipc/RpcScheduler.java | 29 + .../main/java/org/apache/hadoop/ipc/Server.java | 43 +- .../ipc/WeightedRoundRobinMultiplexer.java | 2 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 18 +- .../apache/hadoop/ipc/metrics/RpcMetrics.java | 8 +- .../org/apache/hadoop/jmx/JMXJsonServlet.java | 9 +- .../metrics2/impl/MetricsCollectorImpl.java | 6 +- .../apache/hadoop/metrics2/lib/MutableStat.java | 8 + .../net/NetworkTopologyWithNodeGroup.java | 2 +- .../JniBasedUnixGroupsMappingWithFallback.java | 3 +- .../hadoop/security/LdapGroupsMapping.java | 26 +- .../apache/hadoop/security/NetgroupCache.java | 17 +- .../apache/hadoop/security/SecurityUtil.java | 3 +- .../hadoop/security/WhitelistBasedResolver.java | 149 ++ .../hadoop/security/alias/CredentialShell.java | 65 +- .../hadoop/security/alias/UserProvider.java | 5 - .../authorize/ServiceAuthorizationManager.java | 38 +- .../security/ssl/FileBasedKeyStoresFactory.java | 22 +- .../web/DelegationTokenAuthenticatedURL.java | 398 +++++ .../DelegationTokenAuthenticationFilter.java | 274 ++++ .../DelegationTokenAuthenticationHandler.java | 369 +++++ .../web/DelegationTokenAuthenticator.java | 261 ++++ .../web/DelegationTokenIdentifier.java | 64 + .../delegation/web/DelegationTokenManager.java | 153 ++ .../web/HttpUserGroupInformation.java | 43 + ...rosDelegationTokenAuthenticationHandler.java | 54 + .../KerberosDelegationTokenAuthenticator.java | 46 + ...udoDelegationTokenAuthenticationHandler.java | 55 + .../web/PseudoDelegationTokenAuthenticator.java | 54 + .../token/delegation/web/ServletUtils.java | 59 + .../apache/hadoop/tracing/SpanReceiverHost.java | 153 ++ .../hadoop/util/ApplicationClassLoader.java | 219 +++ .../org/apache/hadoop/util/CacheableIPList.java | 76 + .../java/org/apache/hadoop/util/Classpath.java | 125 ++ .../apache/hadoop/util/CombinedIPWhiteList.java | 60 + .../org/apache/hadoop/util/DataChecksum.java | 22 +- .../org/apache/hadoop/util/DiskChecker.java | 14 + .../org/apache/hadoop/util/FileBasedIPList.java | 102 ++ .../hadoop/util/GenericOptionsParser.java | 6 + .../java/org/apache/hadoop/util/IPList.java | 33 + .../org/apache/hadoop/util/MachineList.java | 13 +- .../apache/hadoop/util/NativeCodeLoader.java | 5 + .../org/apache/hadoop/util/NativeCrc32.java | 43 +- .../hadoop/util/NativeLibraryChecker.java | 21 +- .../apache/hadoop/util/PerformanceAdvisory.java | 25 + .../java/org/apache/hadoop/util/ProtoUtil.java | 11 + .../java/org/apache/hadoop/util/RunJar.java | 115 +- .../org/apache/hadoop/util/StringUtils.java | 14 + .../java/org/apache/hadoop/util/ToolRunner.java | 2 +- .../org/apache/hadoop/util/VersionInfo.java | 3 +- .../hadoop-common/src/main/native/README | 7 +- .../org/apache/hadoop/crypto/OpensslCipher.c | 382 +++++ .../hadoop/crypto/org_apache_hadoop_crypto.h | 61 + .../hadoop/crypto/random/OpensslSecureRandom.c | 335 +++++ .../random/org_apache_hadoop_crypto_random.h | 40 + .../org/apache/hadoop/io/nativeio/NativeIO.c | 72 +- .../apache/hadoop/security/hadoop_user_info.c | 10 + .../org/apache/hadoop/util/NativeCodeLoader.c | 10 + .../src/org/apache/hadoop/util/NativeCrc32.c | 96 +- .../src/org/apache/hadoop/util/bulk_crc32.c | 60 +- .../src/org/apache/hadoop/util/bulk_crc32.h | 49 +- .../org/apache/hadoop/util/test_bulk_crc32.c | 6 +- .../src/main/proto/RpcHeader.proto | 13 + .../src/main/resources/core-default.xml | 88 +- .../src/site/apt/CommandsManual.apt.vm | 380 +---- .../src/site/apt/Compatibility.apt.vm | 9 +- .../src/site/apt/FileSystemShell.apt.vm | 17 +- .../src/site/apt/NativeLibraries.apt.vm | 22 +- .../src/site/apt/ServiceLevelAuth.apt.vm | 21 + .../hadoop-common/src/site/apt/Tracing.apt.vm | 169 +++ .../apache/hadoop/conf/TestConfiguration.java | 8 + .../hadoop/crypto/CryptoStreamsTestBase.java | 721 +++++++++ .../apache/hadoop/crypto/TestCryptoCodec.java | 186 +++ .../apache/hadoop/crypto/TestCryptoStreams.java | 376 +++++ .../crypto/TestCryptoStreamsForLocalFS.java | 120 ++ .../hadoop/crypto/TestCryptoStreamsNormal.java | 123 ++ ...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 31 + .../apache/hadoop/crypto/TestOpensslCipher.java | 110 ++ .../crypto/key/TestCachingKeyProvider.java | 6 + .../hadoop/crypto/key/TestKeyProvider.java | 17 +- .../key/TestKeyProviderCryptoExtension.java | 11 +- ...TestKeyProviderDelegationTokenExtension.java | 13 +- .../crypto/key/TestKeyProviderFactory.java | 79 +- .../apache/hadoop/crypto/key/TestKeyShell.java | 56 +- .../crypto/random/TestOpensslSecureRandom.java | 114 ++ .../crypto/random/TestOsSecureRandom.java | 139 ++ .../apache/hadoop/fs/TestContentSummary.java | 248 ++++ .../org/apache/hadoop/fs/TestHarFileSystem.java | 3 + .../org/apache/hadoop/fs/shell/TestCount.java | 270 ++++ .../apache/hadoop/fs/shell/TestTextCommand.java | 77 +- .../org/apache/hadoop/http/TestHttpServer.java | 6 +- .../hadoop/ipc/TestDecayRpcScheduler.java | 225 +++ .../apache/hadoop/ipc/TestFairCallQueue.java | 392 +++++ .../java/org/apache/hadoop/ipc/TestRPC.java | 8 +- .../hadoop/security/TestLdapGroupsMapping.java | 58 + .../hadoop/security/TestNetgroupCache.java | 127 ++ .../security/TestWhitelistBasedResolver.java | 163 ++ .../hadoop/security/alias/TestCredShell.java | 49 +- .../authorize/TestServiceAuthorization.java | 117 ++ .../hadoop/security/ssl/KeyStoreTestUtil.java | 41 + .../hadoop/security/ssl/TestSSLFactory.java | 58 +- ...tionTokenAuthenticationHandlerWithMocks.java | 364 +++++ .../web/TestDelegationTokenManager.java | 59 + .../delegation/web/TestWebDelegationToken.java | 913 ++++++++++++ .../apache/hadoop/util/ClassLoaderCheck.java | 33 + .../hadoop/util/ClassLoaderCheckMain.java | 34 + .../hadoop/util/ClassLoaderCheckSecond.java | 24 + .../hadoop/util/ClassLoaderCheckThird.java | 24 + .../hadoop/util/TestApplicationClassLoader.java | 136 ++ .../apache/hadoop/util/TestCacheableIPList.java | 188 +++ .../org/apache/hadoop/util/TestClasspath.java | 176 +++ .../apache/hadoop/util/TestDataChecksum.java | 154 +- .../apache/hadoop/util/TestFileBasedIPList.java | 215 +++ .../hadoop/util/TestGenericOptionsParser.java | 66 + .../hadoop/util/TestNativeCodeLoader.java | 4 + .../java/org/apache/hadoop/util/TestRunJar.java | 66 +- .../src/test/resources/testConf.xml | 24 +- hadoop-common-project/hadoop-kms/pom.xml | 1 - .../hadoop/crypto/key/kms/server/KMS.java | 356 +++-- .../hadoop/crypto/key/kms/server/KMSACLs.java | 3 +- .../hadoop/crypto/key/kms/server/KMSAudit.java | 192 ++- .../key/kms/server/KMSAuthenticationFilter.java | 38 +- .../crypto/key/kms/server/KMSConfiguration.java | 9 +- .../key/kms/server/KMSExceptionsProvider.java | 16 +- .../crypto/key/kms/server/KMSJMXServlet.java | 36 + .../crypto/key/kms/server/KMSMDCFilter.java | 17 +- .../hadoop/crypto/key/kms/server/KMSWebApp.java | 20 +- .../hadoop-kms/src/main/webapp/WEB-INF/web.xml | 2 +- .../hadoop-kms/src/site/apt/index.apt.vm | 99 ++ .../hadoop/crypto/key/kms/server/TestKMS.java | 261 +++- .../crypto/key/kms/server/TestKMSACLs.java | 10 +- .../crypto/key/kms/server/TestKMSAudit.java | 135 ++ .../test/resources/log4j-kmsaudit.properties | 25 + .../hadoop/nfs/nfs3/Nfs3FileAttributes.java | 14 +- .../hadoop/nfs/nfs3/request/ACCESS3Request.java | 15 +- .../hadoop/nfs/nfs3/request/COMMIT3Request.java | 23 +- .../hadoop/nfs/nfs3/request/CREATE3Request.java | 18 +- .../hadoop/nfs/nfs3/request/FSINFO3Request.java | 15 +- .../hadoop/nfs/nfs3/request/FSSTAT3Request.java | 15 +- .../nfs/nfs3/request/GETATTR3Request.java | 15 +- .../hadoop/nfs/nfs3/request/LINK3Request.java | 61 + .../hadoop/nfs/nfs3/request/LOOKUP3Request.java | 9 +- .../hadoop/nfs/nfs3/request/MKDIR3Request.java | 26 +- .../hadoop/nfs/nfs3/request/MKNOD3Request.java | 89 ++ .../hadoop/nfs/nfs3/request/NFS3Request.java | 45 + .../nfs/nfs3/request/PATHCONF3Request.java | 15 +- .../hadoop/nfs/nfs3/request/READ3Request.java | 9 +- .../nfs/nfs3/request/READDIR3Request.java | 30 +- .../nfs/nfs3/request/READDIRPLUS3Request.java | 33 +- .../nfs/nfs3/request/READLINK3Request.java | 15 +- .../hadoop/nfs/nfs3/request/REMOVE3Request.java | 20 +- .../hadoop/nfs/nfs3/request/RENAME3Request.java | 37 +- .../hadoop/nfs/nfs3/request/RMDIR3Request.java | 20 +- .../nfs/nfs3/request/RequestWithHandle.java | 16 +- .../nfs/nfs3/request/SETATTR3Request.java | 29 +- .../nfs/nfs3/request/SYMLINK3Request.java | 30 +- .../hadoop/nfs/nfs3/request/SetAttr3.java | 9 + .../hadoop/nfs/nfs3/request/WRITE3Request.java | 13 +- .../hadoop/nfs/nfs3/response/LINK3Response.java | 54 + .../nfs/nfs3/response/MKNOD3Response.java | 68 + hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 1 - .../hadoop/fs/http/client/HttpFSFileSystem.java | 127 +- .../client/HttpFSKerberosAuthenticator.java | 188 --- .../http/client/HttpFSPseudoAuthenticator.java | 47 - .../http/server/HttpFSAuthenticationFilter.java | 25 +- .../HttpFSKerberosAuthenticationHandler.java | 230 --- .../http/server/HttpFSParametersProvider.java | 108 +- .../hadoop/fs/http/server/HttpFSServer.java | 144 +- .../fs/http/server/HttpFSServerWebApp.java | 4 - .../lib/service/DelegationTokenIdentifier.java | 61 - .../lib/service/DelegationTokenManager.java | 78 - .../DelegationTokenManagerException.java | 51 - .../apache/hadoop/lib/service/ProxyUser.java | 31 - .../security/DelegationTokenManagerService.java | 242 --- .../lib/service/security/ProxyUserService.java | 179 --- .../apache/hadoop/lib/wsrs/UserProvider.java | 109 -- .../src/main/resources/httpfs-default.xml | 15 +- ...KerberosAuthenticationHandlerForTesting.java | 6 +- .../http/server/TestHttpFSCustomUserName.java | 94 -- ...TestHttpFSKerberosAuthenticationHandler.java | 316 ---- .../hadoop/fs/http/server/TestHttpFSServer.java | 12 +- .../fs/http/server/TestHttpFSWithKerberos.java | 6 +- .../TestDelegationTokenManagerService.java | 89 -- .../service/security/TestProxyUserService.java | 226 --- .../hadoop/lib/wsrs/TestUserProvider.java | 142 -- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 507 ++++--- .../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 593 ++++++++ .../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 77 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 385 ++++- .../dev-support/findbugsExcludeFile.xml | 9 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 128 +- .../hadoop-hdfs/src/CMakeLists.txt | 78 +- .../bkjournal/BookKeeperEditLogInputStream.java | 5 + .../src/main/bin/distribute-exclude.sh | 6 +- .../hadoop-hdfs/src/main/bin/hdfs | 434 +++--- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 75 +- .../src/main/bin/refresh-namenodes.sh | 40 +- .../hadoop-hdfs/src/main/bin/start-balancer.sh | 30 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 142 +- .../src/main/bin/start-secure-dns.sh | 34 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 30 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 105 +- .../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 34 +- .../src/main/conf/hadoop-metrics2.properties | 44 - .../main/java/org/apache/hadoop/fs/Hdfs.java | 26 +- .../main/java/org/apache/hadoop/fs/XAttr.java | 13 +- .../apache/hadoop/hdfs/BlockReaderFactory.java | 27 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 207 ++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 21 +- .../hadoop/hdfs/DFSInotifyEventInputStream.java | 220 +++ .../org/apache/hadoop/hdfs/DFSInputStream.java | 8 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 92 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 77 +- .../hadoop/hdfs/DistributedFileSystem.java | 81 +- .../org/apache/hadoop/hdfs/StorageType.java | 10 + .../hdfs/UnknownCipherSuiteException.java | 38 + .../org/apache/hadoop/hdfs/XAttrHelper.java | 8 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 100 ++ .../hadoop/hdfs/client/HdfsDataInputStream.java | 38 +- .../hdfs/client/HdfsDataOutputStream.java | 36 +- .../org/apache/hadoop/hdfs/inotify/Event.java | 452 ++++++ .../apache/hadoop/hdfs/inotify/EventsList.java | 63 + .../hdfs/inotify/MissingEventsException.java | 54 + .../org/apache/hadoop/hdfs/protocol/Block.java | 14 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 131 +- .../hadoop/hdfs/protocol/EncryptionZone.java | 79 + .../hdfs/protocol/EncryptionZoneIterator.java | 51 + .../hdfs/protocol/EncryptionZoneWithId.java | 81 + .../protocol/EncryptionZoneWithIdIterator.java | 53 + .../hadoop/hdfs/protocol/HdfsFileStatus.java | 53 +- .../hdfs/protocol/HdfsLocatedFileStatus.java | 10 +- .../hadoop/hdfs/protocol/LocatedBlocks.java | 23 +- .../protocol/SnapshottableDirectoryStatus.java | 2 +- .../datatransfer/ReplaceDatanodeOnFailure.java | 163 +- .../WhitelistBasedTrustedChannelResolver.java | 119 ++ .../datatransfer/sasl/DataTransferSaslUtil.java | 4 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 116 +- .../ClientNamenodeProtocolTranslatorPB.java | 135 +- .../DatanodeProtocolClientSideTranslatorPB.java | 14 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 414 +++++- .../hdfs/qjournal/client/IPCLoggerChannel.java | 78 +- .../hadoop/hdfs/qjournal/server/Journal.java | 3 +- .../hadoop/hdfs/server/balancer/Balancer.java | 1392 ++++-------------- .../hdfs/server/balancer/BalancingPolicy.java | 84 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 1029 +++++++++++++ .../hadoop/hdfs/server/balancer/ExitStatus.java | 44 + .../hadoop/hdfs/server/balancer/KeyManager.java | 173 +++ .../hadoop/hdfs/server/balancer/Matcher.java | 51 + .../hdfs/server/balancer/MovedBlocks.java | 124 ++ .../hdfs/server/balancer/NameNodeConnector.java | 215 +-- .../hdfs/server/blockmanagement/BlockInfo.java | 35 +- .../BlockInfoUnderConstruction.java | 12 +- .../server/blockmanagement/BlockManager.java | 194 ++- .../hdfs/server/blockmanagement/BlocksMap.java | 13 +- .../CacheReplicationMonitor.java | 55 +- .../blockmanagement/CorruptReplicasMap.java | 19 +- .../blockmanagement/DatanodeDescriptor.java | 48 +- .../server/blockmanagement/DatanodeManager.java | 21 +- .../blockmanagement/DatanodeStorageInfo.java | 30 +- .../blockmanagement/HeartbeatManager.java | 8 + .../PendingDataNodeMessages.java | 34 +- .../hdfs/server/common/HdfsServerConstants.java | 5 + .../hdfs/server/datanode/BPOfferService.java | 305 ++-- .../hdfs/server/datanode/BPServiceActor.java | 15 +- .../server/datanode/BlockPoolSliceStorage.java | 56 +- .../hdfs/server/datanode/BlockReceiver.java | 59 +- .../hdfs/server/datanode/BlockSender.java | 6 +- .../hadoop/hdfs/server/datanode/DataNode.java | 51 +- .../server/datanode/DataNodeLayoutVersion.java | 5 +- .../hdfs/server/datanode/DataStorage.java | 352 ++++- .../hdfs/server/datanode/DataXceiver.java | 26 +- .../hdfs/server/datanode/DataXceiverServer.java | 40 +- .../hdfs/server/datanode/DatanodeUtil.java | 36 + .../hdfs/server/datanode/ReplicaInfo.java | 62 +- .../server/datanode/ShortCircuitRegistry.java | 29 +- .../hdfs/server/datanode/StorageLocation.java | 2 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 9 + .../datanode/fsdataset/impl/BlockPoolSlice.java | 98 +- .../impl/FsDatasetAsyncDiskService.java | 87 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 162 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 56 +- .../datanode/fsdataset/impl/FsVolumeList.java | 41 +- .../server/datanode/fsdataset/impl/LDir.java | 228 --- .../namenode/EditLogBackupInputStream.java | 5 + .../server/namenode/EditLogFileInputStream.java | 5 + .../server/namenode/EditLogInputStream.java | 6 + .../namenode/EncryptionFaultInjector.java | 39 + .../server/namenode/EncryptionZoneManager.java | 313 ++++ .../hdfs/server/namenode/FSDirectory.java | 324 +++- .../hadoop/hdfs/server/namenode/FSEditLog.java | 67 +- .../hdfs/server/namenode/FSEditLogLoader.java | 3 +- .../hdfs/server/namenode/FSImageFormat.java | 11 + .../hdfs/server/namenode/FSNamesystem.java | 769 ++++++++-- .../server/namenode/FileJournalManager.java | 47 +- .../hadoop/hdfs/server/namenode/INode.java | 59 +- .../hdfs/server/namenode/INodeDirectory.java | 5 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 12 +- .../hadoop/hdfs/server/namenode/INodeMap.java | 3 +- .../hdfs/server/namenode/INodeReference.java | 4 +- .../hdfs/server/namenode/INodeSymlink.java | 3 +- .../namenode/InotifyFSEditLogOpTranslator.java | 146 ++ .../hadoop/hdfs/server/namenode/JournalSet.java | 38 +- .../hadoop/hdfs/server/namenode/NameNode.java | 8 + .../hdfs/server/namenode/NameNodeRpcServer.java | 165 ++- .../namenode/RedundantEditLogInputStream.java | 5 + .../namenode/RetryStartFileException.java | 21 + .../hdfs/server/namenode/TransferFsImage.java | 5 +- .../server/namenode/XAttrPermissionFilter.java | 30 +- .../namenode/metrics/FSNamesystemMBean.java | 7 + .../snapshot/DirectoryWithSnapshotFeature.java | 10 +- .../web/resources/NamenodeWebHdfsMethods.java | 26 +- .../server/protocol/BlocksWithLocations.java | 25 +- .../server/protocol/DatanodeStorageReport.java | 42 + .../hdfs/server/protocol/RegisterCommand.java | 3 + .../hadoop/hdfs/shortcircuit/DfsClientShm.java | 21 +- .../hdfs/shortcircuit/DfsClientShmManager.java | 10 +- .../hdfs/shortcircuit/DomainSocketFactory.java | 4 +- .../hdfs/shortcircuit/ShortCircuitShm.java | 10 +- .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 301 ++++ .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +- .../offlineEditsViewer/XmlEditsVisitor.java | 4 +- .../apache/hadoop/hdfs/util/EnumCounters.java | 9 +- .../apache/hadoop/hdfs/util/EnumDoubles.java | 128 ++ .../org/apache/hadoop/hdfs/web/JsonUtil.java | 5 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 7 + .../hdfs/web/resources/FsActionParam.java | 58 + .../hadoop/hdfs/web/resources/GetOpParam.java | 4 +- .../src/main/native/fuse-dfs/CMakeLists.txt | 4 + .../src/main/native/libhdfs/common/htable.c | 271 ++++ .../src/main/native/libhdfs/common/htable.h | 161 ++ .../src/main/native/libhdfs/exception.c | 65 +- .../src/main/native/libhdfs/exception.h | 10 +- .../src/main/native/libhdfs/expect.c | 8 +- .../src/main/native/libhdfs/expect.h | 12 + .../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 416 +++--- .../src/main/native/libhdfs/jni_helper.c | 265 ++-- .../src/main/native/libhdfs/jni_helper.h | 2 - .../src/main/native/libhdfs/native_mini_dfs.c | 6 +- .../src/main/native/libhdfs/os/mutexes.h | 55 + .../src/main/native/libhdfs/os/posix/mutexes.c | 43 + .../src/main/native/libhdfs/os/posix/platform.h | 34 + .../src/main/native/libhdfs/os/posix/thread.c | 52 + .../libhdfs/os/posix/thread_local_storage.c | 80 + .../src/main/native/libhdfs/os/thread.h | 54 + .../native/libhdfs/os/thread_local_storage.h | 75 + .../main/native/libhdfs/os/windows/inttypes.h | 28 + .../main/native/libhdfs/os/windows/mutexes.c | 52 + .../main/native/libhdfs/os/windows/platform.h | 86 ++ .../src/main/native/libhdfs/os/windows/thread.c | 66 + .../libhdfs/os/windows/thread_local_storage.c | 164 +++ .../src/main/native/libhdfs/os/windows/unistd.h | 29 + .../main/native/libhdfs/test/test_libhdfs_ops.c | 232 +-- .../native/libhdfs/test/test_libhdfs_read.c | 19 +- .../native/libhdfs/test/test_libhdfs_write.c | 29 +- .../native/libhdfs/test/test_libhdfs_zerocopy.c | 31 +- .../main/native/libhdfs/test_libhdfs_threaded.c | 39 +- .../main/native/libhdfs/test_native_mini_dfs.c | 2 +- .../src/main/proto/ClientNamenodeProtocol.proto | 53 + .../src/main/proto/DatanodeProtocol.proto | 24 - .../src/main/proto/datatransfer.proto | 1 + .../hadoop-hdfs/src/main/proto/encryption.proto | 65 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 47 +- .../hadoop-hdfs/src/main/proto/inotify.proto | 117 ++ .../hadoop-hdfs/src/main/proto/xattr.proto | 1 + .../src/main/resources/hdfs-default.xml | 56 +- .../src/main/webapps/hdfs/dfshealth.html | 5 +- .../src/main/webapps/hdfs/dfshealth.js | 11 +- .../src/main/webapps/hdfs/index.html | 13 +- .../src/main/webapps/secondary/index.html | 11 - .../src/site/apt/ExtendedAttributes.apt.vm | 4 +- .../src/site/apt/HDFSCommands.apt.vm | 408 +++++ .../src/site/apt/HdfsNfsGateway.apt.vm | 73 +- .../src/site/apt/HdfsUserGuide.apt.vm | 29 +- .../hadoop-hdfs/src/site/apt/LibHdfs.apt.vm | 29 +- .../src/site/apt/TransparentEncryption.apt.vm | 206 +++ .../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 44 + .../src/site/xdoc/HdfsRollingUpgrade.xml | 2 +- .../org/apache/hadoop/TestGenericRefresh.java | 29 +- .../org/apache/hadoop/TestRefreshCallQueue.java | 44 +- .../apache/hadoop/cli/TestCryptoAdminCLI.java | 169 +++ .../hadoop/cli/util/CLICommandCryptoAdmin.java | 21 + .../hadoop/cli/util/CryptoAdminCmdExecutor.java | 37 + .../org/apache/hadoop/fs/TestGlobPaths.java | 260 ++-- .../java/org/apache/hadoop/fs/TestXAttr.java | 11 +- .../fs/viewfs/TestViewFileSystemWithXAttrs.java | 151 ++ .../hadoop/fs/viewfs/TestViewFsWithXAttrs.java | 148 ++ .../org/apache/hadoop/hdfs/DFSTestUtil.java | 100 ++ .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 243 ++- .../hdfs/MiniDFSClusterWithNodeGroup.java | 38 +- .../hadoop/hdfs/TestDFSClientFailover.java | 11 + .../hadoop/hdfs/TestDFSClientRetries.java | 11 +- .../org/apache/hadoop/hdfs/TestDFSFinalize.java | 9 +- .../hdfs/TestDFSInotifyEventInputStream.java | 430 ++++++ .../apache/hadoop/hdfs/TestDFSPermission.java | 77 + .../org/apache/hadoop/hdfs/TestDFSRename.java | 44 + .../org/apache/hadoop/hdfs/TestDFSRollback.java | 2 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 177 ++- .../hdfs/TestDFSStorageStateRecovery.java | 16 +- .../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 12 +- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 154 +- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 75 +- .../hadoop/hdfs/TestDatanodeBlockScanner.java | 15 +- .../apache/hadoop/hdfs/TestDatanodeConfig.java | 2 + .../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 48 + .../apache/hadoop/hdfs/TestDatanodeReport.java | 90 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 1 - .../apache/hadoop/hdfs/TestEncryptionZones.java | 756 ++++++++++ .../org/apache/hadoop/hdfs/TestFileAppend.java | 61 +- .../apache/hadoop/hdfs/TestFileCorruption.java | 40 +- .../apache/hadoop/hdfs/TestFileCreation.java | 2 +- .../java/org/apache/hadoop/hdfs/TestLease.java | 9 +- .../hdfs/TestReplaceDatanodeOnFailure.java | 45 +- .../hadoop/hdfs/TestReservedRawPaths.java | 350 +++++ .../org/apache/hadoop/hdfs/TestSafeMode.java | 25 +- .../hdfs/TestWriteBlockGetsBlockLengthHint.java | 106 ++ .../apache/hadoop/hdfs/UpgradeUtilities.java | 44 +- .../hdfs/crypto/TestHdfsCryptoStreams.java | 91 ++ .../hadoop/hdfs/protocolPB/TestPBHelper.java | 21 +- .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 6 +- .../hdfs/qjournal/client/TestQJMWithFaults.java | 2 +- .../client/TestQuorumJournalManager.java | 2 +- .../security/token/block/TestBlockToken.java | 2 + .../hdfs/server/balancer/TestBalancer.java | 567 ++++++- .../balancer/TestBalancerWithHANameNodes.java | 8 +- .../TestBalancerWithMultipleNameNodes.java | 8 +- .../balancer/TestBalancerWithNodeGroup.java | 12 +- .../blockmanagement/BlockManagerTestUtil.java | 10 + .../server/blockmanagement/TestBlockInfo.java | 20 +- .../blockmanagement/TestBlockManager.java | 10 +- .../blockmanagement/TestCorruptReplicaInfo.java | 14 +- .../blockmanagement/TestDatanodeDescriptor.java | 6 +- .../TestPendingDataNodeMessages.java | 7 +- .../blockmanagement/TestReplicationPolicy.java | 5 +- .../server/datanode/SimulatedFSDataset.java | 11 + .../TestBlockHasMultipleReplicasOnSameDN.java | 5 +- .../hdfs/server/datanode/TestBlockRecovery.java | 1 - .../server/datanode/TestBlockReplacement.java | 6 +- .../datanode/TestDataNodeRollingUpgrade.java | 48 + .../datanode/TestDataNodeVolumeFailure.java | 24 +- .../hdfs/server/datanode/TestDataStorage.java | 210 +++ .../server/datanode/TestDeleteBlockPool.java | 7 +- .../hdfs/server/datanode/TestDiskError.java | 4 +- ...TestDnRespectsBlockReportSplitThreshold.java | 6 +- .../server/datanode/TestFsDatasetCache.java | 1 - .../hdfs/server/datanode/TestStorageReport.java | 2 +- .../fsdataset/impl/TestFsDatasetImpl.java | 183 +++ .../hdfs/server/namenode/FSAclBaseTest.java | 28 + .../hdfs/server/namenode/FSXAttrBaseTest.java | 179 ++- .../server/namenode/NNThroughputBenchmark.java | 5 +- .../hdfs/server/namenode/TestAddBlockRetry.java | 4 +- .../namenode/TestBlockUnderConstruction.java | 3 + .../server/namenode/TestCacheDirectives.java | 1 - .../TestCommitBlockSynchronization.java | 11 + .../namenode/TestDecommissioningStatus.java | 98 +- .../hdfs/server/namenode/TestDeleteRace.java | 133 ++ .../hdfs/server/namenode/TestEditLog.java | 4 + .../hdfs/server/namenode/TestFSDirectory.java | 15 +- .../server/namenode/TestFSNamesystemMBean.java | 2 + .../hadoop/hdfs/server/namenode/TestFsck.java | 19 +- .../hdfs/server/namenode/TestINodeFile.java | 16 + .../namenode/TestListCorruptFileBlocks.java | 163 +- .../namenode/TestNNStorageRetentionManager.java | 11 +- .../server/namenode/TestNameNodeMXBean.java | 45 +- .../server/namenode/TestNamenodeRetryCache.java | 9 +- .../hdfs/server/namenode/TestStartup.java | 41 + .../namenode/ha/TestPipelinesFailover.java | 62 +- .../namenode/ha/TestRetryCacheWithHA.java | 10 +- .../namenode/metrics/TestNameNodeMetrics.java | 2 +- .../namenode/snapshot/TestAclWithSnapshot.java | 9 + .../namenode/snapshot/TestSnapshotDeletion.java | 77 +- .../shortcircuit/TestShortCircuitCache.java | 98 ++ .../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 39 +- .../hadoop/hdfs/web/TestWebHDFSForHA.java | 76 +- .../hdfs/web/TestWebHdfsFileSystemContract.java | 33 + .../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 24 + .../hadoop/security/TestPermissionSymlinks.java | 35 + .../security/TestRefreshUserMappings.java | 12 +- .../org/apache/hadoop/tracing/TestTracing.java | 280 ++++ .../src/test/resources/hadoop-0.23-reserved.tgz | Bin 0 -> 4558 bytes .../src/test/resources/hadoop-1-reserved.tgz | Bin 0 -> 2572 bytes .../test/resources/hadoop-24-datanode-dir.tgz | Bin 0 -> 637608 bytes .../src/test/resources/hadoop-datanode-dir.txt | 23 + .../src/test/resources/testCryptoConf.xml | 284 ++++ .../src/test/resources/testHDFSConf.xml | 44 + .../src/test/resources/testXAttrConf.xml | 58 +- hadoop-mapreduce-project/CHANGES.txt | 109 +- hadoop-mapreduce-project/bin/mapred | 226 +-- hadoop-mapreduce-project/bin/mapred-config.sh | 76 +- .../bin/mr-jobhistory-daemon.sh | 143 +- hadoop-mapreduce-project/conf/mapred-env.sh | 60 +- .../hadoop-mapreduce-client-app/pom.xml | 6 + .../hadoop/mapred/LocalContainerLauncher.java | 18 +- .../hadoop/mapred/TaskAttemptListenerImpl.java | 4 +- .../org/apache/hadoop/mapred/YarnChild.java | 22 +- .../jobhistory/JobHistoryEventHandler.java | 12 +- .../hadoop/mapreduce/v2/app/AppContext.java | 1 + .../hadoop/mapreduce/v2/app/MRAppMaster.java | 5 + .../v2/app/client/MRClientService.java | 3 +- .../v2/app/commit/CommitterEventHandler.java | 2 +- .../mapreduce/v2/app/job/impl/JobImpl.java | 30 +- .../mapreduce/v2/app/rm/RMCommunicator.java | 2 +- .../v2/app/rm/RMContainerAllocator.java | 26 +- .../v2/app/speculate/DefaultSpeculator.java | 8 +- .../mapreduce/v2/app/webapp/TaskPage.java | 3 +- .../mapreduce/v2/app/webapp/TasksBlock.java | 4 +- .../mapred/TestTaskAttemptListenerImpl.java | 13 +- .../jobhistory/TestJobHistoryEventHandler.java | 127 +- .../hadoop/mapreduce/v2/app/MockAppContext.java | 5 + .../mapreduce/v2/app/TestJobEndNotifier.java | 3 +- .../mapreduce/v2/app/TestRuntimeEstimators.java | 5 + .../mapreduce/v2/app/job/impl/TestJobImpl.java | 10 +- .../mapreduce/v2/app/webapp/TestBlocks.java | 3 + .../mapreduce/v2/jobhistory/JHAdminConfig.java | 4 +- .../v2/jobhistory/JobHistoryUtils.java | 79 +- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 3 +- .../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 9 +- .../hadoop/mapreduce/v2/util/TestMRApps.java | 5 +- .../org/apache/hadoop/mapred/BackupStore.java | 8 +- .../java/org/apache/hadoop/mapred/IFile.java | 21 +- .../org/apache/hadoop/mapred/InputFormat.java | 2 +- .../java/org/apache/hadoop/mapred/JobConf.java | 94 +- .../java/org/apache/hadoop/mapred/MapTask.java | 70 +- .../java/org/apache/hadoop/mapred/Merger.java | 15 +- .../java/org/apache/hadoop/mapred/Task.java | 6 + .../hadoop/mapred/TaskCompletionEvent.java | 4 +- .../mapred/join/CompositeInputFormat.java | 2 +- .../apache/hadoop/mapreduce/CryptoUtils.java | 199 +++ .../apache/hadoop/mapreduce/InputFormat.java | 2 +- .../java/org/apache/hadoop/mapreduce/Job.java | 11 +- .../apache/hadoop/mapreduce/JobSubmitter.java | 9 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 23 + .../hadoop/mapreduce/TaskCompletionEvent.java | 4 +- .../mapreduce/lib/db/OracleDBRecordReader.java | 7 +- .../lib/input/KeyValueTextInputFormat.java | 3 + .../lib/join/CompositeInputFormat.java | 2 +- .../hadoop/mapreduce/task/reduce/Fetcher.java | 14 +- .../mapreduce/task/reduce/LocalFetcher.java | 4 + .../mapreduce/task/reduce/MergeManagerImpl.java | 37 +- .../mapreduce/task/reduce/OnDiskMapOutput.java | 3 +- .../task/reduce/ShuffleSchedulerImpl.java | 34 +- .../hadoop/mapreduce/util/ConfigUtil.java | 2 + .../src/main/resources/mapred-default.xml | 107 +- .../src/site/apt/MapredCommands.apt.vm | 230 +++ .../apt/PluggableShuffleAndPluggableSort.apt.vm | 8 +- .../src/site/markdown/DistCp.md.vm | 28 +- .../src/site/markdown/HadoopArchives.md.vm | 27 +- .../org/apache/hadoop/mapred/TestJobConf.java | 15 +- .../hadoop/mapreduce/lib/db/TestDbClasses.java | 2 +- .../mapreduce/task/reduce/TestMerger.java | 146 +- .../mapreduce/v2/hs/HistoryClientService.java | 11 +- ...istoryServerFileSystemStateStoreService.java | 101 +- .../hadoop/mapreduce/v2/hs/JobHistory.java | 5 + .../mapreduce/v2/hs/server/HSAdminServer.java | 5 +- .../mapreduce/v2/hs/webapp/HsTaskPage.java | 3 +- .../mapreduce/v2/hs/TestHistoryFileManager.java | 70 +- ...istoryServerFileSystemStateStoreService.java | 85 +- .../mapreduce/v2/hs/webapp/TestBlocks.java | 4 +- .../org/apache/hadoop/mapred/YARNRunner.java | 9 + .../test/java/org/apache/hadoop/SleepJob.java | 275 ---- .../org/apache/hadoop/conf/TestJobConf.java | 20 +- .../org/apache/hadoop/fs/JHLogAnalyzer.java | 4 +- .../apache/hadoop/mapred/TestComparators.java | 61 +- .../org/apache/hadoop/mapred/TestIFile.java | 11 +- .../TestMRIntermediateDataEncryption.java | 254 ++++ .../apache/hadoop/mapred/TestMapOutputType.java | 62 +- .../org/apache/hadoop/mapred/TestMapRed.java | 28 +- .../apache/hadoop/mapred/TestReduceTask.java | 2 +- .../apache/hadoop/mapred/TestYARNRunner.java | 104 +- .../mapred/lib/TestKeyFieldBasedComparator.java | 25 +- .../mapred/pipes/TestPipeApplication.java | 12 +- .../org/apache/hadoop/mapreduce/SleepJob.java | 32 +- .../apache/hadoop/mapreduce/TestMapReduce.java | 30 +- .../TestMRAMWithNonNormalizedCapabilities.java | 2 +- .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 9 +- .../v2/TestMRJobsWithHistoryService.java | 2 +- .../mapreduce/v2/TestMRJobsWithProfiler.java | 2 +- .../apache/hadoop/mapred/ShuffleHandler.java | 28 +- .../hadoop/mapred/TestShuffleHandler.java | 8 +- hadoop-project-dist/pom.xml | 14 + hadoop-project/pom.xml | 49 +- hadoop-project/src/site/site.xml | 5 + .../org/apache/hadoop/tools/HadoopArchives.java | 14 +- .../apache/hadoop/tools/TestHadoopArchives.java | 41 + .../hadoop-aws/dev-support/findbugs-exclude.xml | 19 + hadoop-tools/hadoop-aws/pom.xml | 116 ++ .../apache/hadoop/tools/DistCpConstants.java | 6 + .../apache/hadoop/tools/DistCpOptionSwitch.java | 6 +- .../org/apache/hadoop/tools/DistCpOptions.java | 21 +- .../apache/hadoop/tools/SimpleCopyListing.java | 48 +- .../hadoop/tools/mapred/CopyCommitter.java | 9 +- .../apache/hadoop/tools/mapred/CopyMapper.java | 10 +- .../apache/hadoop/tools/util/DistCpUtils.java | 48 +- .../hadoop/tools/TestDistCpWithRawXAttrs.java | 170 +++ .../hadoop/tools/TestDistCpWithXAttrs.java | 71 +- .../apache/hadoop/tools/TestOptionsParser.java | 3 +- .../hadoop/tools/util/DistCpTestUtils.java | 89 ++ .../hadoop/tools/util/TestDistCpUtils.java | 6 +- .../mapred/gridmix/TestGridmixRecord.java | 10 +- .../hadoop/mapred/gridmix/TestHighRamJob.java | 12 +- hadoop-tools/hadoop-openstack/pom.xml | 5 + .../hadoop/yarn/sls/RumenToSLSConverter.java | 4 + .../org/apache/hadoop/yarn/sls/SLSRunner.java | 4 + .../hadoop/yarn/sls/appmaster/AMSimulator.java | 73 +- .../yarn/sls/appmaster/MRAMSimulator.java | 9 +- .../hadoop/yarn/sls/conf/SLSConfiguration.java | 5 + .../yarn/sls/nodemanager/NMSimulator.java | 72 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 35 +- .../sls/scheduler/CapacitySchedulerMetrics.java | 5 + .../yarn/sls/scheduler/ContainerSimulator.java | 4 + .../sls/scheduler/FairSchedulerMetrics.java | 12 +- .../sls/scheduler/FifoSchedulerMetrics.java | 4 + .../NodeUpdateSchedulerEventWrapper.java | 4 + .../yarn/sls/scheduler/RMNodeWrapper.java | 15 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 50 +- .../sls/scheduler/SLSCapacityScheduler.java | 26 +- .../yarn/sls/scheduler/SchedulerMetrics.java | 4 + .../yarn/sls/scheduler/SchedulerWrapper.java | 4 + .../hadoop/yarn/sls/scheduler/TaskRunner.java | 22 +- .../apache/hadoop/yarn/sls/utils/SLSUtils.java | 4 + .../apache/hadoop/yarn/sls/web/SLSWebApp.java | 4 + .../apache/hadoop/yarn/sls/TestSLSRunner.java | 30 +- .../yarn/sls/appmaster/TestAMSimulator.java | 86 ++ .../yarn/sls/nodemanager/TestNMSimulator.java | 100 ++ hadoop-tools/pom.xml | 1 + hadoop-yarn-project/CHANGES.txt | 194 ++- hadoop-yarn-project/hadoop-yarn/bin/slaves.sh | 70 - .../hadoop-yarn/bin/start-yarn.sh | 34 +- .../hadoop-yarn/bin/stop-yarn.sh | 40 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 398 ++--- .../hadoop-yarn/bin/yarn-config.sh | 120 +- .../hadoop-yarn/bin/yarn-daemon.sh | 157 +- .../hadoop-yarn/bin/yarn-daemons.sh | 38 +- .../hadoop-yarn/conf/yarn-env.sh | 163 +- .../dev-support/findbugs-exclude.xml | 14 + .../api/protocolrecords/AllocateResponse.java | 32 + .../protocolrecords/GetApplicationsRequest.java | 18 + .../hadoop/yarn/api/records/ContainerId.java | 10 +- .../hadoop/yarn/api/records/ResourceOption.java | 4 + .../hadoop/yarn/conf/YarnConfiguration.java | 40 +- .../UpdateNodeResourceResponse.java | 12 +- ..._server_resourcemanager_service_protos.proto | 63 - .../src/main/proto/yarn_service_protos.proto | 1 + .../distributedshell/ApplicationMaster.java | 31 +- .../applications/distributedshell/Client.java | 2 +- .../distributedshell/TestDSFailedAppMaster.java | 6 +- .../distributedshell/TestDistributedShell.java | 75 +- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 8 - .../hadoop/yarn/client/api/AMRMClient.java | 63 + .../hadoop/yarn/client/api/TimelineClient.java | 88 -- .../yarn/client/api/async/AMRMClientAsync.java | 64 + .../yarn/client/api/impl/AMRMClientImpl.java | 19 + .../client/api/impl/TimelineAuthenticator.java | 260 ---- .../client/api/impl/TimelineClientImpl.java | 332 ----- .../hadoop/yarn/client/ProtocolHATestBase.java | 1 + .../TestApplicationMasterServiceOnHA.java | 6 +- .../api/async/impl/TestAMRMClientAsync.java | 76 +- .../yarn/client/api/impl/TestAMRMClient.java | 170 ++- .../api/impl/TestAMRMClientOnRMRestart.java | 151 ++ .../api/impl/TestTimelineAuthenticator.java | 40 - .../client/api/impl/TestTimelineClient.java | 206 --- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 8 + .../impl/pb/AllocateResponsePBImpl.java | 38 +- .../impl/pb/GetApplicationsRequestPBImpl.java | 10 + .../impl/pb/ApplicationReportPBImpl.java | 1 + .../pb/ApplicationSubmissionContextPBImpl.java | 1 + .../impl/pb/ResourceBlacklistRequestPBImpl.java | 13 +- .../records/impl/pb/ResourceOptionPBImpl.java | 48 +- .../yarn/api/records/impl/pb/TokenPBImpl.java | 2 +- .../hadoop/yarn/client/api/TimelineClient.java | 88 ++ .../client/api/impl/TimelineAuthenticator.java | 269 ++++ .../client/api/impl/TimelineClientImpl.java | 406 +++++ .../yarn/client/api/impl/package-info.java | 21 + .../hadoop/yarn/client/api/package-info.java | 21 + .../pb/UpdateNodeResourceRequestPBImpl.java | 16 +- .../pb/UpdateNodeResourceResponsePBImpl.java | 2 +- .../yarn/util/ApplicationClassLoader.java | 170 +-- .../resource/DefaultResourceCalculator.java | 7 + .../resource/DominantResourceCalculator.java | 8 + .../yarn/util/resource/ResourceCalculator.java | 9 + .../hadoop/yarn/util/resource/Resources.java | 5 + .../hadoop/yarn/webapp/util/WebAppUtils.java | 83 +- .../src/main/resources/yarn-default.xml | 68 +- .../apache/hadoop/yarn/api/TestContainerId.java | 4 +- .../hadoop/yarn/api/TestPBImplRecords.java | 895 +++++++++++ .../api/impl/TestTimelineAuthenticator.java | 40 + .../client/api/impl/TestTimelineClient.java | 206 +++ .../hadoop/yarn/conf/TestYarnConfiguration.java | 128 ++ .../yarn/util/TestApplicationClassLoader.java | 136 -- .../yarn/webapp/util/TestWebAppUtils.java | 148 ++ .../ApplicationHistoryClientService.java | 16 +- .../ApplicationHistoryServer.java | 117 +- .../FileSystemApplicationHistoryStore.java | 14 +- .../webapp/AHSWebApp.java | 25 +- .../server/timeline/LeveldbTimelineStore.java | 74 +- .../server/timeline/TimelineDataManager.java | 319 ++++ .../timeline/webapp/CrossOriginFilter.java | 220 +++ .../webapp/CrossOriginFilterInitializer.java | 42 + .../timeline/webapp/TimelineWebServices.java | 255 +--- .../TestApplicationHistoryClientService.java | 2 +- .../TestApplicationHistoryServer.java | 41 + .../TestFileSystemApplicationHistoryStore.java | 62 +- .../timeline/TestLeveldbTimelineStore.java | 53 +- .../timeline/webapp/TestCrossOriginFilter.java | 214 +++ .../TestCrossOriginFilterInitializer.java | 60 + .../webapp/TestTimelineWebServices.java | 9 +- .../webapp/TestTimelineWebServicesWithSSL.java | 134 ++ .../hadoop/yarn/server/records/Version.java | 84 ++ .../server/records/impl/pb/VersionPBImpl.java | 77 + .../security/http/RMAuthenticationFilter.java | 63 + .../http/RMAuthenticationFilterInitializer.java | 121 ++ .../main/proto/yarn_server_common_protos.proto | 8 +- .../server/nodemanager/ContainerExecutor.java | 73 +- .../nodemanager/DefaultContainerExecutor.java | 59 +- .../nodemanager/LinuxContainerExecutor.java | 25 +- .../server/nodemanager/NodeStatusUpdater.java | 23 + .../nodemanager/NodeStatusUpdaterImpl.java | 34 +- .../containermanager/ContainerManagerImpl.java | 260 +++- .../application/ApplicationImpl.java | 6 + .../container/ContainerImpl.java | 165 ++- .../launcher/ContainerLaunch.java | 46 +- .../launcher/ContainersLauncher.java | 15 +- .../launcher/ContainersLauncherEventType.java | 1 + .../launcher/RecoveredContainerLaunch.java | 126 ++ .../localizer/ResourceLocalizationService.java | 10 +- .../logaggregation/AppLogAggregator.java | 2 + .../logaggregation/AppLogAggregatorImpl.java | 14 +- .../logaggregation/LogAggregationService.java | 10 +- .../recovery/NMLeveldbStateStoreService.java | 287 +++- .../recovery/NMNullStateStoreService.java | 58 + .../recovery/NMStateStoreService.java | 212 +++ .../recovery/records/NMDBSchemaVersion.java | 80 - .../impl/pb/NMDBSchemaVersionPBImpl.java | 81 - .../server/nodemanager/webapp/WebServer.java | 4 +- .../impl/container-executor.c | 111 +- .../yarn_server_nodemanager_recovery.proto | 12 +- .../nodemanager/TestLinuxContainerExecutor.java | 7 + .../nodemanager/TestNodeManagerResync.java | 2 +- .../nodemanager/TestNodeStatusUpdater.java | 17 +- .../BaseContainerManagerTest.java | 2 +- .../containermanager/TestAuxServices.java | 3 +- .../TestContainerManagerRecovery.java | 324 ++++ .../container/TestContainer.java | 6 +- .../recovery/NMMemoryStateStoreService.java | 152 +- .../TestNMLeveldbStateStoreService.java | 187 ++- .../nodemanager/webapp/TestNMWebServer.java | 2 +- .../conf/capacity-scheduler.xml | 23 + .../hadoop-yarn-server-resourcemanager/pom.xml | 31 + .../server/resourcemanager/AdminService.java | 50 +- .../ApplicationMasterService.java | 41 +- .../server/resourcemanager/ClientRMService.java | 8 +- .../resourcemanager/EmbeddedElectorService.java | 10 +- .../resourcemanager/RMSecretManagerService.java | 6 +- .../server/resourcemanager/ResourceManager.java | 103 +- .../resourcemanager/ResourceTrackerService.java | 14 +- .../resourcemanager/amlauncher/AMLauncher.java | 11 +- .../recovery/FileSystemRMStateStore.java | 87 +- .../recovery/MemoryRMStateStore.java | 21 +- .../recovery/NullRMStateStore.java | 13 +- .../resourcemanager/recovery/RMStateStore.java | 108 +- .../recovery/RMStateStoreFactory.java | 13 +- .../recovery/ZKRMStateStore.java | 78 +- .../records/AMRMTokenSecretManagerState.java | 76 + .../records/ApplicationAttemptStateData.java | 2 +- .../recovery/records/ApplicationStateData.java | 2 +- .../resourcemanager/recovery/records/Epoch.java | 2 +- .../recovery/records/RMStateVersion.java | 80 - .../pb/AMRMTokenSecretManagerStatePBImpl.java | 126 ++ .../pb/ApplicationAttemptStateDataPBImpl.java | 6 +- .../impl/pb/ApplicationStateDataPBImpl.java | 6 +- .../recovery/records/impl/pb/EpochPBImpl.java | 4 +- .../records/impl/pb/RMStateVersionPBImpl.java | 76 - .../server/resourcemanager/rmapp/RMAppImpl.java | 40 +- .../rmapp/RMAppNewSavedEvent.java | 36 - .../rmapp/RMAppUpdateSavedEvent.java | 36 - .../rmapp/attempt/RMAppAttemptImpl.java | 84 +- .../event/RMAppAttemptNewSavedEvent.java | 39 - .../event/RMAppAttemptUpdateSavedEvent.java | 38 - .../server/resourcemanager/rmnode/RMNode.java | 12 - .../resourcemanager/rmnode/RMNodeEventType.java | 3 + .../resourcemanager/rmnode/RMNodeImpl.java | 162 +- .../rmnode/RMNodeReconnectEvent.java | 12 +- .../rmnode/RMNodeResourceUpdateEvent.java | 37 + .../scheduler/AbstractYarnScheduler.java | 185 +++ .../scheduler/AppSchedulingInfo.java | 5 +- .../scheduler/SchedulerApplicationAttempt.java | 18 +- .../scheduler/SchedulerNode.java | 25 +- .../scheduler/SchedulerUtils.java | 32 - .../scheduler/YarnScheduler.java | 18 + .../scheduler/capacity/CSQueue.java | 18 + .../scheduler/capacity/CSQueueUtils.java | 53 + .../scheduler/capacity/CapacityScheduler.java | 223 ++- .../CapacitySchedulerConfiguration.java | 119 +- .../scheduler/capacity/LeafQueue.java | 50 +- .../scheduler/capacity/ParentQueue.java | 33 + .../event/NodeResourceUpdateSchedulerEvent.java | 43 + .../scheduler/event/SchedulerEventType.java | 1 + .../scheduler/fair/AllocationConfiguration.java | 40 +- .../fair/AllocationFileLoaderService.java | 57 +- .../scheduler/fair/AppSchedulable.java | 463 ------ .../scheduler/fair/FSAppAttempt.java | 762 ++++++++++ .../scheduler/fair/FSLeafQueue.java | 85 +- .../scheduler/fair/FSOpDurations.java | 119 ++ .../scheduler/fair/FSParentQueue.java | 25 +- .../resourcemanager/scheduler/fair/FSQueue.java | 85 +- .../scheduler/fair/FSQueueMetrics.java | 17 +- .../scheduler/fair/FSSchedulerApp.java | 360 ----- .../scheduler/fair/FSSchedulerNode.java | 6 +- .../scheduler/fair/FairScheduler.java | 227 +-- .../scheduler/fair/FifoAppComparator.java | 8 +- .../scheduler/fair/MaxRunningAppsEnforcer.java | 52 +- .../scheduler/fair/NewAppWeightBooster.java | 2 +- .../scheduler/fair/QueueManager.java | 37 +- .../scheduler/fair/Schedulable.java | 61 +- .../scheduler/fair/SchedulingPolicy.java | 27 +- .../scheduler/fair/WeightAdjuster.java | 2 +- .../fair/policies/ComputeFairShares.java | 55 +- .../DominantResourceFairnessPolicy.java | 9 + .../fair/policies/FairSharePolicy.java | 8 + .../scheduler/fair/policies/FifoPolicy.java | 8 + .../scheduler/fifo/FifoScheduler.java | 42 +- .../security/AMRMTokenSecretManager.java | 53 +- .../security/DelegationTokenRenewer.java | 2 +- .../security/RMAuthenticationHandler.java | 157 ++ .../resourcemanager/webapp/AboutBlock.java | 1 + .../webapp/FairSchedulerAppsBlock.java | 4 + .../webapp/FairSchedulerPage.java | 31 +- .../resourcemanager/webapp/RMWebServices.java | 26 +- .../resourcemanager/webapp/dao/ClusterInfo.java | 8 + .../webapp/dao/FairSchedulerInfo.java | 16 +- .../webapp/dao/FairSchedulerLeafQueueInfo.java | 9 +- .../webapp/dao/FairSchedulerQueueInfo.java | 34 +- .../yarn_server_resourcemanager_recovery.proto | 90 ++ .../yarn/server/resourcemanager/MockAM.java | 8 +- .../yarn/server/resourcemanager/MockNodes.java | 22 +- .../yarn/server/resourcemanager/MockRM.java | 15 +- .../MockRMWithCustomAMLauncher.java | 5 +- .../resourcemanager/TestApplicationCleanup.java | 29 + .../TestApplicationMasterService.java | 1 - .../resourcemanager/TestFifoScheduler.java | 85 ++ .../yarn/server/resourcemanager/TestRMHA.java | 46 +- .../resourcemanager/TestRMNodeTransitions.java | 80 +- .../server/resourcemanager/TestRMRestart.java | 25 +- .../resourcemanager/TestResourceManager.java | 76 + .../TestResourceTrackerService.java | 2 +- .../TestWorkPreservingRMRestart.java | 101 +- .../applicationsmanager/TestAMRestart.java | 9 +- .../recovery/RMStateStoreTestBase.java | 108 +- .../recovery/TestFSRMStateStore.java | 14 +- .../recovery/TestZKRMStateStore.java | 13 +- .../resourcetracker/TestNMReconnect.java | 19 +- .../rmapp/TestRMAppTransitions.java | 15 +- .../attempt/TestRMAppAttemptTransitions.java | 48 +- .../scheduler/capacity/TestCSQueueUtils.java | 262 ++++ .../capacity/TestCapacityScheduler.java | 983 ++++++++++++- .../scheduler/capacity/TestQueueMappings.java | 240 +++ .../scheduler/capacity/TestUtils.java | 5 +- .../scheduler/fair/FakeSchedulable.java | 13 +- .../fair/TestAllocationFileLoaderService.java | 64 +- .../scheduler/fair/TestFSAppAttempt.java | 188 +++ .../scheduler/fair/TestFSLeafQueue.java | 2 +- .../scheduler/fair/TestFSSchedulerApp.java | 191 --- .../scheduler/fair/TestFairScheduler.java | 684 ++++++++- .../fair/TestFairSchedulerFairShare.java | 360 +++++ .../fair/TestMaxRunningAppsEnforcer.java | 42 +- .../scheduler/fifo/TestFifoScheduler.java | 24 +- .../security/TestAMRMTokens.java | 55 +- .../webapp/TestRMWebAppFairScheduler.java | 96 +- .../webapp/TestRMWebServices.java | 2 +- .../TestRMWebServicesAppsModification.java | 21 +- ...ebServicesDelegationTokenAuthentication.java | 357 +++++ .../webapp/TestRMWebappAuthentication.java | 272 ++++ .../src/site/apt/FairScheduler.apt.vm | 38 +- .../src/site/apt/ResourceManagerRest.apt.vm | 21 + .../src/site/apt/WritingYarnApplications.apt.vm | 1369 +++++++++-------- 977 files changed, 60546 insertions(+), 16363 deletions(-) ----------------------------------------------------------------------