Merge branch 'apache-trunk' into HDFS-6994
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a607429b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a607429b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a607429b Branch: refs/heads/HDFS-6994 Commit: a607429b55b26dd7ab75d3fa4d3c87072d542c50 Parents: 2e42564 08ac062 Author: cnauroth <[email protected]> Authored: Tue Jan 13 09:44:54 2015 -0800 Committer: cnauroth <[email protected]> Committed: Tue Jan 13 09:44:54 2015 -0800 ---------------------------------------------------------------------- .gitignore | 7 + BUILDING.txt | 25 +- LICENSE.txt | 7 +- dev-support/create-release.sh | 6 + dev-support/smart-apply-patch.sh | 47 + dev-support/test-patch.sh | 73 - hadoop-assemblies/pom.xml | 4 +- hadoop-client/pom.xml | 2 +- .../hadoop-annotations/pom.xml | 14 +- .../authentication/examples/WhoClient.java | 5 +- .../authentication/client/AuthenticatedURL.java | 13 +- .../authentication/client/Authenticator.java | 2 +- .../client/KerberosAuthenticator.java | 16 +- .../client/PseudoAuthenticator.java | 10 +- .../AltKerberosAuthenticationHandler.java | 5 +- .../server/AuthenticationFilter.java | 79 +- .../server/AuthenticationHandler.java | 24 +- .../server/AuthenticationToken.java | 4 +- .../server/KerberosAuthenticationHandler.java | 13 +- .../server/PseudoAuthenticationHandler.java | 26 +- .../authentication/util/KerberosName.java | 4 +- .../authentication/util/KerberosUtil.java | 8 +- .../util/RandomSignerSecretProvider.java | 4 +- .../security/authentication/util/Signer.java | 5 +- .../util/StringSignerSecretProvider.java | 3 +- .../util/ZKSignerSecretProvider.java | 11 +- .../client/TestPseudoAuthenticator.java | 5 +- .../server/TestAuthenticationFilter.java | 6 +- .../server/TestPseudoAuthenticationHandler.java | 9 +- hadoop-common-project/hadoop-common/CHANGES.txt | 518 ++- .../dev-support/findbugsExcludeFile.xml | 18 +- hadoop-common-project/hadoop-common/pom.xml | 17 +- .../hadoop-common/src/CMakeLists.txt | 76 +- .../hadoop-common/src/main/bin/hadoop | 11 +- .../src/main/bin/hadoop-config.cmd | 16 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 9 + .../src/main/bin/hadoop-daemons.sh | 19 +- .../src/main/bin/hadoop-functions.sh | 158 +- .../hadoop-common/src/main/bin/hadoop.cmd | 29 +- .../hadoop-common/src/main/bin/rcc | 2 - .../hadoop-common/src/main/bin/slaves.sh | 2 +- .../hadoop-common/src/main/bin/stop-all.sh | 2 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 146 +- .../main/conf/hadoop-user-functions.sh.example | 7 - .../src/main/docs/releasenotes.html | 3569 ++++++++++++++++++ .../org/apache/hadoop/conf/Configuration.java | 85 +- .../apache/hadoop/conf/ReconfigurableBase.java | 2 +- .../hadoop/conf/ReconfigurationServlet.java | 4 +- .../apache/hadoop/crypto/AesCtrCryptoCodec.java | 27 +- .../org/apache/hadoop/crypto/CipherOption.java | 66 + .../apache/hadoop/crypto/CryptoInputStream.java | 46 +- .../hadoop/crypto/CryptoOutputStream.java | 1 + .../apache/hadoop/crypto/CryptoStreamUtils.java | 7 + .../crypto/UnsupportedCodecException.java | 60 + .../hadoop/crypto/key/JavaKeyStoreProvider.java | 12 +- .../apache/hadoop/crypto/key/KeyProvider.java | 14 +- .../crypto/key/KeyProviderCryptoExtension.java | 9 +- .../org/apache/hadoop/crypto/key/KeyShell.java | 32 +- .../crypto/key/kms/KMSClientProvider.java | 89 +- .../hadoop/crypto/key/kms/ValueQueue.java | 24 +- .../apache/hadoop/fs/AbstractFileSystem.java | 11 +- .../apache/hadoop/fs/ByteBufferReadable.java | 11 +- .../apache/hadoop/fs/ChecksumFileSystem.java | 21 +- .../org/apache/hadoop/fs/FSOutputSummer.java | 17 +- .../java/org/apache/hadoop/fs/FileContext.java | 17 +- .../java/org/apache/hadoop/fs/FileSystem.java | 33 +- .../java/org/apache/hadoop/fs/FileUtil.java | 32 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 7 + .../org/apache/hadoop/fs/HarFileSystem.java | 23 +- .../org/apache/hadoop/fs/LocalDirAllocator.java | 2 +- .../hadoop/fs/MD5MD5CRC32FileChecksum.java | 8 +- .../main/java/org/apache/hadoop/fs/Options.java | 20 +- .../main/java/org/apache/hadoop/fs/Path.java | 1 - .../apache/hadoop/fs/RawLocalFileSystem.java | 125 +- .../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 21 +- .../fs/permission/AccessControlException.java | 66 - .../apache/hadoop/fs/permission/AclEntry.java | 4 +- .../apache/hadoop/fs/permission/AclStatus.java | 79 +- .../org/apache/hadoop/fs/shell/AclCommands.java | 32 +- .../org/apache/hadoop/fs/shell/Command.java | 28 +- .../apache/hadoop/fs/shell/CommandFactory.java | 1 + .../org/apache/hadoop/fs/shell/Display.java | 12 +- .../org/apache/hadoop/fs/shell/FsCommand.java | 2 + .../org/apache/hadoop/fs/shell/FsUsage.java | 17 +- .../java/org/apache/hadoop/fs/shell/Ls.java | 2 +- .../java/org/apache/hadoop/fs/shell/Stat.java | 4 +- .../java/org/apache/hadoop/fs/shell/Test.java | 2 + .../org/apache/hadoop/fs/shell/find/And.java | 84 + .../hadoop/fs/shell/find/BaseExpression.java | 302 ++ .../apache/hadoop/fs/shell/find/Expression.java | 107 + .../hadoop/fs/shell/find/ExpressionFactory.java | 156 + .../hadoop/fs/shell/find/FilterExpression.java | 144 + .../org/apache/hadoop/fs/shell/find/Find.java | 444 +++ .../hadoop/fs/shell/find/FindOptions.java | 271 ++ .../org/apache/hadoop/fs/shell/find/Name.java | 100 + .../org/apache/hadoop/fs/shell/find/Print.java | 76 + .../org/apache/hadoop/fs/shell/find/Result.java | 88 + .../apache/hadoop/ha/ActiveStandbyElector.java | 4 +- .../main/java/org/apache/hadoop/ha/HAAdmin.java | 6 - .../org/apache/hadoop/ha/SshFenceByTcpPort.java | 2 + .../java/org/apache/hadoop/ha/StreamPumper.java | 4 +- .../org/apache/hadoop/http/HtmlQuoting.java | 19 +- .../org/apache/hadoop/http/HttpServer2.java | 150 +- .../apache/hadoop/io/DefaultStringifier.java | 3 +- .../main/java/org/apache/hadoop/io/IOUtils.java | 36 + .../java/org/apache/hadoop/io/LongWritable.java | 4 +- .../java/org/apache/hadoop/io/SequenceFile.java | 4 +- .../main/java/org/apache/hadoop/io/Text.java | 2 + .../apache/hadoop/io/WritableComparator.java | 1 + .../apache/hadoop/io/compress/BZip2Codec.java | 7 +- .../apache/hadoop/io/compress/CodecPool.java | 54 +- .../hadoop/io/compress/DecompressorStream.java | 2 +- .../org/apache/hadoop/io/file/tfile/TFile.java | 2 +- .../hadoop/io/file/tfile/TFileDumper.java | 11 +- .../org/apache/hadoop/io/nativeio/Errno.java | 1 + .../org/apache/hadoop/io/nativeio/NativeIO.java | 126 +- .../main/java/org/apache/hadoop/ipc/Client.java | 2 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 4 +- .../org/apache/hadoop/ipc/RpcClientUtil.java | 21 + .../org/apache/hadoop/ipc/RpcConstants.java | 4 +- .../main/java/org/apache/hadoop/ipc/Server.java | 20 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 4 +- .../org/apache/hadoop/jmx/JMXJsonServlet.java | 46 +- .../java/org/apache/hadoop/log/LogLevel.java | 3 +- .../apache/hadoop/metrics/file/FileContext.java | 159 - .../hadoop/metrics/ganglia/GangliaContext.java | 3 +- .../metrics/ganglia/GangliaContext31.java | 5 - .../hadoop/metrics/spi/CompositeContext.java | 2 +- .../hadoop/metrics2/impl/MetricsConfig.java | 8 +- .../metrics2/impl/MetricsSourceAdapter.java | 5 +- .../hadoop/metrics2/lib/MutableCounterInt.java | 17 +- .../hadoop/metrics2/lib/MutableCounterLong.java | 19 +- .../hadoop/metrics2/lib/MutableGaugeInt.java | 30 +- .../hadoop/metrics2/lib/MutableGaugeLong.java | 30 +- .../apache/hadoop/metrics2/sink/FileSink.java | 12 +- .../hadoop/metrics2/sink/GraphiteSink.java | 145 +- .../sink/ganglia/AbstractGangliaSink.java | 10 +- .../java/org/apache/hadoop/net/NetUtils.java | 12 +- .../net/ScriptBasedMappingWithDependency.java | 3 +- .../org/apache/hadoop/net/TableMapping.java | 19 +- .../hadoop/net/unix/DomainSocketWatcher.java | 15 + .../hadoop/security/AccessControlException.java | 4 +- .../AuthenticationFilterInitializer.java | 11 +- .../org/apache/hadoop/security/Credentials.java | 4 +- .../java/org/apache/hadoop/security/Groups.java | 205 +- .../hadoop/security/IdMappingConstant.java | 36 + .../security/IdMappingServiceProvider.java | 56 + .../hadoop/security/LdapGroupsMapping.java | 14 +- .../apache/hadoop/security/NetgroupCache.java | 61 +- .../apache/hadoop/security/SaslRpcServer.java | 8 +- .../apache/hadoop/security/SecurityUtil.java | 17 + .../hadoop/security/ShellBasedIdMapping.java | 712 ++++ .../hadoop/security/UserGroupInformation.java | 43 +- .../hadoop/security/alias/CredentialShell.java | 20 +- .../security/alias/JavaKeyStoreProvider.java | 5 +- .../hadoop/security/alias/UserProvider.java | 4 +- .../security/authorize/AccessControlList.java | 6 +- .../authorize/AuthorizationException.java | 2 +- .../authorize/ServiceAuthorizationManager.java | 58 +- .../apache/hadoop/security/ssl/SSLFactory.java | 12 +- .../security/ssl/SslSocketConnectorSecure.java | 58 + .../AbstractDelegationTokenIdentifier.java | 58 +- .../AbstractDelegationTokenSecretManager.java | 76 +- .../ZKDelegationTokenSecretManager.java | 212 +- .../web/DelegationTokenAuthenticatedURL.java | 76 +- .../DelegationTokenAuthenticationFilter.java | 22 +- .../DelegationTokenAuthenticationHandler.java | 58 +- .../web/DelegationTokenAuthenticator.java | 75 +- .../delegation/web/DelegationTokenManager.java | 60 +- .../apache/hadoop/tracing/SpanReceiverHost.java | 16 +- .../org/apache/hadoop/tracing/TraceAdmin.java | 9 +- .../hadoop/tracing/TraceSamplerFactory.java | 2 +- .../hadoop/util/ApplicationClassLoader.java | 34 +- .../apache/hadoop/util/ChunkedArrayList.java | 205 + .../java/org/apache/hadoop/util/Classpath.java | 2 +- .../apache/hadoop/util/ComparableVersion.java | 2 + .../org/apache/hadoop/util/DataChecksum.java | 18 +- .../org/apache/hadoop/util/FileBasedIPList.java | 55 +- .../org/apache/hadoop/util/HostsFileReader.java | 4 +- .../apache/hadoop/util/IdentityHashStore.java | 14 +- .../org/apache/hadoop/util/JvmPauseMonitor.java | 6 +- .../apache/hadoop/util/LimitInputStream.java | 109 + .../org/apache/hadoop/util/MachineList.java | 3 +- .../hadoop/util/NativeLibraryChecker.java | 14 +- .../apache/hadoop/util/PrintJarMainClass.java | 17 +- .../org/apache/hadoop/util/ReflectionUtils.java | 14 +- .../org/apache/hadoop/util/ServletUtil.java | 87 +- .../main/java/org/apache/hadoop/util/Shell.java | 40 +- .../java/org/apache/hadoop/util/StopWatch.java | 108 + .../src/main/native/native.vcxproj | 4 + .../src/org/apache/hadoop/io/compress/lz4/lz4.c | 1243 ++++-- .../src/org/apache/hadoop/io/compress/lz4/lz4.h | 296 +- .../apache/hadoop/io/compress/lz4/lz4_encoder.h | 258 -- .../org/apache/hadoop/io/compress/lz4/lz4hc.c | 591 ++- .../org/apache/hadoop/io/compress/lz4/lz4hc.h | 84 +- .../hadoop/io/compress/lz4/lz4hc_encoder.h | 349 -- .../org/apache/hadoop/io/nativeio/NativeIO.c | 122 +- .../org/apache/hadoop/io/nativeio/errno_enum.c | 1 + .../windows_secure_container_executor.c | 569 +++ .../windows_secure_container_executor.h | 27 + .../src/main/resources/core-default.xml | 10 +- .../hadoop-common/src/main/winutils/chown.c | 130 +- .../hadoop-common/src/main/winutils/client.c | 498 +++ .../hadoop-common/src/main/winutils/config.cpp | 174 + .../src/main/winutils/hadoopwinutilsvc.idl | 130 + .../src/main/winutils/include/winutils.h | 120 +- .../src/main/winutils/libwinutils.c | 1019 ++++- .../src/main/winutils/libwinutils.vcxproj | 90 +- .../hadoop-common/src/main/winutils/main.c | 23 + .../hadoop-common/src/main/winutils/service.c | 1485 ++++++++ .../hadoop-common/src/main/winutils/task.c | 1211 ++++-- .../hadoop-common/src/main/winutils/winutils.mc | 64 + .../src/main/winutils/winutils.sln | 18 +- .../src/main/winutils/winutils.vcxproj | 106 +- .../src/site/apt/ClusterSetup.apt.vm | 348 +- .../src/site/apt/CommandsManual.apt.vm | 312 +- .../src/site/apt/FileSystemShell.apt.vm | 350 +- .../hadoop-common/src/site/apt/Metrics.apt.vm | 139 + .../src/site/apt/NativeLibraries.apt.vm | 18 + .../src/site/apt/RackAwareness.apt.vm | 140 + .../src/site/apt/SecureMode.apt.vm | 96 +- .../src/site/apt/ServiceLevelAuth.apt.vm | 25 + .../src/site/apt/SingleCluster.apt.vm | 20 +- .../hadoop-common/src/site/apt/Tracing.apt.vm | 77 +- .../markdown/filesystem/fsdatainputstream.md | 62 +- .../src/site/markdown/filesystem/testing.md | 47 - .../conf/TestConfigurationFieldsBase.java | 417 ++ .../apache/hadoop/conf/TestReconfiguration.java | 5 +- .../apache/hadoop/crypto/TestCryptoCodec.java | 104 +- .../crypto/key/TestKeyProviderFactory.java | 34 +- .../apache/hadoop/crypto/key/TestKeyShell.java | 3 +- .../crypto/random/TestOsSecureRandom.java | 14 - .../fs/FileContextCreateMkdirBaseTest.java | 11 +- .../hadoop/fs/FileSystemContractBaseTest.java | 6 +- .../apache/hadoop/fs/FileSystemTestWrapper.java | 28 +- .../org/apache/hadoop/fs/SymlinkBaseTest.java | 3 +- .../org/apache/hadoop/fs/TestFileContext.java | 41 + .../java/org/apache/hadoop/fs/TestFileUtil.java | 10 +- .../org/apache/hadoop/fs/TestHarFileSystem.java | 1 + .../fs/contract/AbstractContractDeleteTest.java | 27 + .../fs/contract/AbstractContractMkdirTest.java | 19 + .../fs/contract/AbstractContractRenameTest.java | 41 + .../hadoop/fs/contract/ContractOptions.java | 7 + .../hadoop/fs/contract/ContractTestUtils.java | 139 + .../hadoop/fs/loadGenerator/LoadGenerator.java | 343 +- .../hadoop/fs/shell/find/MockFileSystem.java | 86 + .../apache/hadoop/fs/shell/find/TestAnd.java | 263 ++ .../fs/shell/find/TestFilterExpression.java | 145 + .../apache/hadoop/fs/shell/find/TestFind.java | 900 +++++ .../apache/hadoop/fs/shell/find/TestHelper.java | 35 + .../apache/hadoop/fs/shell/find/TestIname.java | 93 + .../apache/hadoop/fs/shell/find/TestName.java | 93 + .../apache/hadoop/fs/shell/find/TestPrint.java | 56 + .../apache/hadoop/fs/shell/find/TestPrint0.java | 56 + .../apache/hadoop/fs/shell/find/TestResult.java | 172 + .../org/apache/hadoop/http/TestHttpServer.java | 78 +- .../java/org/apache/hadoop/io/TestIOUtils.java | 44 + .../java/org/apache/hadoop/io/TestUTF8.java | 23 +- .../hadoop/io/compress/TestCodecPool.java | 47 +- .../apache/hadoop/io/nativeio/TestNativeIO.java | 35 + .../apache/hadoop/jmx/TestJMXJsonServlet.java | 48 +- .../metrics2/impl/TestGraphiteMetrics.java | 129 +- .../metrics2/impl/TestMetricsSourceAdapter.java | 51 + .../org/apache/hadoop/net/TestNetUtils.java | 12 + .../hadoop/net/TestSocketIOWithTimeout.java | 7 +- .../hadoop/security/TestGroupsCaching.java | 284 ++ .../hadoop/security/TestSecurityUtil.java | 4 - .../security/TestShellBasedIdMapping.java | 367 ++ .../hadoop/security/TestUGILoginFromKeytab.java | 91 + .../hadoop/security/alias/TestCredShell.java | 7 +- .../authorize/TestServiceAuthorization.java | 189 +- .../hadoop/security/ssl/KeyStoreTestUtil.java | 73 +- .../TestZKDelegationTokenSecretManager.java | 297 +- ...tionTokenAuthenticationHandlerWithMocks.java | 12 +- .../web/TestDelegationTokenManager.java | 21 +- .../delegation/web/TestWebDelegationToken.java | 56 +- .../apache/hadoop/test/GenericTestUtils.java | 48 + .../hadoop/util/TestApplicationClassLoader.java | 34 +- .../hadoop/util/TestChunkedArrayList.java | 170 + .../apache/hadoop/util/TestDataChecksum.java | 10 +- .../org/apache/hadoop/util/TestStopWatch.java | 62 + .../org/apache/hadoop/util/TestWinUtils.java | 2 +- .../src/test/resources/contract/localfs.xml | 4 + .../resources/javakeystoreprovider.password | 2 +- .../src/test/resources/testConf.xml | 46 +- hadoop-common-project/hadoop-kms/pom.xml | 10 +- .../hadoop-kms/src/main/conf/kms-env.sh | 43 +- .../hadoop/crypto/key/kms/server/KMSACLs.java | 49 +- .../crypto/key/kms/server/KMSConfiguration.java | 1 + .../crypto/key/kms/server/KMSJSONWriter.java | 4 +- .../kms/server/KeyAuthorizationKeyProvider.java | 143 +- .../hadoop-kms/src/main/libexec/kms-config.sh | 203 +- .../hadoop-kms/src/main/sbin/kms.sh | 109 +- .../hadoop-kms/src/main/tomcat/ssl-server.xml | 135 - .../src/main/tomcat/ssl-server.xml.conf | 136 + .../hadoop-kms/src/site/apt/index.apt.vm | 47 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 5 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 430 ++- .../java/org/apache/hadoop/minikdc/MiniKdc.java | 10 +- .../org/apache/hadoop/minikdc/TestMiniKdc.java | 22 +- .../org/apache/hadoop/mount/MountResponse.java | 3 +- .../org/apache/hadoop/nfs/nfs3/FileHandle.java | 7 +- .../org/apache/hadoop/nfs/nfs3/IdUserGroup.java | 392 -- .../org/apache/hadoop/nfs/nfs3/Nfs3Base.java | 5 +- .../apache/hadoop/nfs/nfs3/Nfs3Constant.java | 12 - .../hadoop/nfs/nfs3/Nfs3FileAttributes.java | 29 +- .../hadoop/nfs/nfs3/request/CREATE3Request.java | 3 +- .../hadoop/nfs/nfs3/request/LINK3Request.java | 3 +- .../hadoop/nfs/nfs3/request/LOOKUP3Request.java | 5 +- .../hadoop/nfs/nfs3/request/MKDIR3Request.java | 5 +- .../hadoop/nfs/nfs3/request/MKNOD3Request.java | 3 +- .../hadoop/nfs/nfs3/request/REMOVE3Request.java | 5 +- .../hadoop/nfs/nfs3/request/RENAME3Request.java | 9 +- .../hadoop/nfs/nfs3/request/RMDIR3Request.java | 5 +- .../nfs/nfs3/request/SYMLINK3Request.java | 9 +- .../org/apache/hadoop/oncrpc/RpcProgram.java | 7 +- .../java/org/apache/hadoop/oncrpc/RpcUtil.java | 2 +- .../main/java/org/apache/hadoop/oncrpc/XDR.java | 5 +- .../hadoop/oncrpc/security/CredentialsSys.java | 3 +- .../oncrpc/security/SysSecurityHandler.java | 10 +- .../apache/hadoop/nfs/nfs3/TestIdUserGroup.java | 217 -- hadoop-dist/pom.xml | 2 +- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 6 + .../hadoop/fs/http/client/HttpFSFileSystem.java | 24 +- .../fs/http/client/HttpsFSFileSystem.java | 8 +- .../server/CheckUploadContentTypeFilter.java | 4 +- .../http/server/HttpFSAuthenticationFilter.java | 2 +- .../fs/http/server/HttpFSExceptionProvider.java | 1 - .../hadoop/fs/http/server/HttpFSServer.java | 26 +- .../fs/http/server/HttpFSServerWebApp.java | 4 +- .../org/apache/hadoop/lib/lang/XException.java | 4 +- .../apache/hadoop/lib/server/BaseService.java | 16 +- .../org/apache/hadoop/lib/server/Server.java | 42 +- .../lib/servlet/FileSystemReleaseFilter.java | 6 +- .../hadoop/lib/servlet/HostnameFilter.java | 6 +- .../apache/hadoop/lib/servlet/MDCFilter.java | 6 +- .../apache/hadoop/lib/servlet/ServerWebApp.java | 20 +- .../java/org/apache/hadoop/lib/util/Check.java | 2 +- .../hadoop/lib/util/ConfigurationUtils.java | 2 +- .../org/apache/hadoop/lib/wsrs/Parameters.java | 4 +- .../src/main/tomcat/ssl-server.xml | 2 +- .../org/apache/hadoop/test/TestJettyHelper.java | 3 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 + .../hadoop/hdfs/nfs/conf/NfsConfigKeys.java | 16 + .../hadoop/hdfs/nfs/conf/NfsConfiguration.java | 7 +- .../hadoop/hdfs/nfs/nfs3/DFSClientCache.java | 2 +- .../org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java | 3 +- .../hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java | 111 + .../hadoop/hdfs/nfs/nfs3/Nfs3Metrics.java | 220 ++ .../apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java | 38 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 221 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java | 4 +- .../nfs/nfs3/PrivilegedNfsGatewayStarter.java | 2 +- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 174 +- .../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 18 +- .../hadoop/hdfs/nfs/nfs3/WriteManager.java | 30 +- .../hdfs/nfs/nfs3/TestDFSClientCache.java | 5 +- .../hdfs/nfs/nfs3/TestNfs3HttpServer.java | 93 + .../hdfs/nfs/nfs3/TestOpenFileCtxCache.java | 20 +- .../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 5 +- .../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 200 +- .../src/test/resources/core-site.xml | 14 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 776 +++- .../dev-support/findbugsExcludeFile.xml | 9 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 11 +- .../hadoop-hdfs/src/CMakeLists.txt | 8 +- .../src/ant/org/apache/hadoop/ant/DfsTask.java | 205 - .../src/ant/org/apache/hadoop/ant/antlib.xml | 29 - .../ant/condition/DfsBaseConditional.java | 68 - .../apache/hadoop/ant/condition/DfsExists.java | 24 - .../apache/hadoop/ant/condition/DfsIsDir.java | 24 - .../apache/hadoop/ant/condition/DfsZeroLen.java | 24 - .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 18 - .../bkjournal/TestBookKeeperHACheckpoints.java | 2 +- .../hadoop-hdfs/src/main/bin/hdfs | 40 +- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 7 +- .../hadoop-hdfs/src/main/bin/hdfs.cmd | 14 +- .../hadoop-hdfs/src/main/bin/start-balancer.sh | 4 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 2 +- .../main/java/org/apache/hadoop/fs/Hdfs.java | 3 - .../apache/hadoop/hdfs/BlockReaderFactory.java | 15 +- .../apache/hadoop/hdfs/BlockReaderLocal.java | 29 +- .../hadoop/hdfs/BlockReaderLocalLegacy.java | 33 +- .../org/apache/hadoop/hdfs/ClientContext.java | 10 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 125 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 69 +- .../hadoop/hdfs/DFSInotifyEventInputStream.java | 65 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 418 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 296 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 15 +- .../hadoop/hdfs/DistributedFileSystem.java | 200 +- .../java/org/apache/hadoop/hdfs/PeerCache.java | 12 +- .../hadoop/hdfs/client/BlockReportOptions.java | 59 + .../org/apache/hadoop/hdfs/inotify/Event.java | 87 +- .../apache/hadoop/hdfs/inotify/EventBatch.java | 41 + .../hadoop/hdfs/inotify/EventBatchList.java | 63 + .../apache/hadoop/hdfs/inotify/EventsList.java | 63 - .../java/org/apache/hadoop/hdfs/package.html | 2 +- .../hdfs/protocol/ClientDatanodeProtocol.java | 7 + .../hadoop/hdfs/protocol/ClientProtocol.java | 49 +- .../hadoop/hdfs/protocol/HdfsConstants.java | 14 + .../hdfs/protocol/LastBlockWithStatus.java | 46 + .../hadoop/hdfs/protocol/LayoutVersion.java | 4 +- .../hadoop/hdfs/protocol/LocatedBlock.java | 7 +- .../hadoop/hdfs/protocol/LocatedBlocks.java | 9 +- .../protocol/datatransfer/PacketReceiver.java | 2 +- .../datatransfer/sasl/DataTransferSaslUtil.java | 278 +- .../sasl/InvalidMagicNumberException.java | 14 +- .../sasl/SaslDataTransferClient.java | 61 +- .../sasl/SaslDataTransferServer.java | 56 +- .../datatransfer/sasl/SaslParticipant.java | 44 + .../SaslResponseWithNegotiatedCipherOption.java | 33 + ...tDatanodeProtocolServerSideTranslatorPB.java | 18 + .../ClientDatanodeProtocolTranslatorPB.java | 16 + ...tNamenodeProtocolServerSideTranslatorPB.java | 34 +- .../ClientNamenodeProtocolTranslatorPB.java | 31 +- ...rDatanodeProtocolServerSideTranslatorPB.java | 6 +- .../InterDatanodeProtocolTranslatorPB.java | 5 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 466 ++- .../hdfs/qjournal/client/IPCLoggerChannel.java | 8 +- .../qjournal/server/GetJournalEditServlet.java | 21 +- .../hadoop/hdfs/qjournal/server/Journal.java | 30 +- .../hdfs/qjournal/server/JournalNode.java | 1 + .../hadoop/hdfs/server/balancer/Balancer.java | 81 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 32 +- .../hdfs/server/balancer/NameNodeConnector.java | 6 + .../server/blockmanagement/BlockIdManager.java | 208 + .../hdfs/server/blockmanagement/BlockInfo.java | 9 +- .../BlockInfoUnderConstruction.java | 22 +- .../server/blockmanagement/BlockManager.java | 262 +- .../blockmanagement/BlockPlacementPolicy.java | 1 - .../BlockPlacementPolicyDefault.java | 49 +- .../BlockPlacementPolicyWithNodeGroup.java | 1 - .../BlockStoragePolicySuite.java | 39 +- .../hdfs/server/blockmanagement/BlocksMap.java | 6 +- .../CacheReplicationMonitor.java | 12 +- .../blockmanagement/CorruptReplicasMap.java | 23 +- .../blockmanagement/DatanodeDescriptor.java | 122 +- .../server/blockmanagement/DatanodeManager.java | 125 +- .../blockmanagement/DatanodeStorageInfo.java | 26 +- .../server/blockmanagement/FSClusterStats.java | 60 + .../blockmanagement/HeartbeatManager.java | 52 +- .../blockmanagement/MutableBlockCollection.java | 38 - .../SequentialBlockIdGenerator.java | 66 + .../blockmanagement/UnderReplicatedBlocks.java | 35 +- .../hdfs/server/common/HdfsServerConstants.java | 16 +- .../hadoop/hdfs/server/common/JspHelper.java | 5 +- .../hadoop/hdfs/server/common/Storage.java | 113 +- .../hadoop/hdfs/server/common/StorageInfo.java | 6 +- .../hdfs/server/datanode/BPOfferService.java | 4 +- .../hdfs/server/datanode/BPServiceActor.java | 103 +- .../server/datanode/BlockMetadataHeader.java | 43 +- .../server/datanode/BlockPoolSliceStorage.java | 168 +- .../hdfs/server/datanode/BlockReceiver.java | 146 +- .../hdfs/server/datanode/BlockSender.java | 57 +- .../hadoop/hdfs/server/datanode/DNConf.java | 11 + .../hadoop/hdfs/server/datanode/DataNode.java | 361 +- .../server/datanode/DataNodeFaultInjector.java | 4 + .../hdfs/server/datanode/DataNodeMXBean.java | 7 + .../hdfs/server/datanode/DataStorage.java | 561 ++- .../hdfs/server/datanode/DataXceiver.java | 209 +- .../hdfs/server/datanode/DataXceiverServer.java | 2 +- .../hdfs/server/datanode/DatanodeUtil.java | 21 + .../hdfs/server/datanode/ReplicaInPipeline.java | 7 +- .../datanode/ReplicaNotFoundException.java | 2 +- .../server/datanode/SecureDataNodeStarter.java | 48 +- .../server/datanode/ShortCircuitRegistry.java | 2 +- .../UnexpectedReplicaStateException.java | 45 + .../server/datanode/fsdataset/FsDatasetSpi.java | 59 +- .../fsdataset/ReplicaOutputStreams.java | 9 +- .../datanode/fsdataset/impl/BlockPoolSlice.java | 50 +- .../datanode/fsdataset/impl/FsDatasetCache.java | 4 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 696 ++-- .../datanode/fsdataset/impl/FsVolumeImpl.java | 19 +- .../datanode/fsdataset/impl/FsVolumeList.java | 190 +- .../impl/RamDiskAsyncLazyPersistService.java | 252 ++ .../impl/RamDiskReplicaLruTracker.java | 4 +- .../datanode/metrics/DataNodeMetrics.java | 7 + .../server/datanode/web/DatanodeHttpServer.java | 174 + .../datanode/web/SimpleHttpProxyHandler.java | 161 + .../hdfs/server/datanode/web/URLDispatcher.java | 59 + .../web/resources/DatanodeWebHdfsMethods.java | 477 --- .../datanode/web/resources/OpenEntity.java | 93 - .../web/webhdfs/DataNodeUGIProvider.java | 70 + .../datanode/web/webhdfs/ExceptionHandler.java | 116 + .../server/datanode/web/webhdfs/HdfsWriter.java | 82 + .../datanode/web/webhdfs/ParameterParser.java | 125 + .../datanode/web/webhdfs/WebHdfsHandler.java | 258 ++ .../apache/hadoop/hdfs/server/mover/Mover.java | 34 +- .../server/namenode/AclEntryStatusFormat.java | 136 + .../hadoop/hdfs/server/namenode/AclFeature.java | 61 +- .../hadoop/hdfs/server/namenode/AclStorage.java | 91 +- .../namenode/DirectoryWithQuotaFeature.java | 3 +- .../namenode/EditLogFileOutputStream.java | 6 +- .../server/namenode/EncryptionZoneManager.java | 21 +- .../hdfs/server/namenode/FSClusterStats.java | 62 - .../hadoop/hdfs/server/namenode/FSDirAclOp.java | 249 ++ .../hdfs/server/namenode/FSDirAttrOp.java | 459 +++ .../hdfs/server/namenode/FSDirConcatOp.java | 233 ++ .../hdfs/server/namenode/FSDirMkdirOp.java | 244 ++ .../hdfs/server/namenode/FSDirRenameOp.java | 761 ++++ .../hdfs/server/namenode/FSDirSnapshotOp.java | 238 ++ .../server/namenode/FSDirStatAndListingOp.java | 501 +++ .../hdfs/server/namenode/FSDirSymlinkOp.java | 129 + .../hdfs/server/namenode/FSDirXAttrOp.java | 463 +++ .../hdfs/server/namenode/FSDirectory.java | 2938 ++++---------- .../hadoop/hdfs/server/namenode/FSEditLog.java | 19 + .../hdfs/server/namenode/FSEditLogLoader.java | 175 +- .../hdfs/server/namenode/FSEditLogOp.java | 426 ++- .../hdfs/server/namenode/FSEditLogOpCodes.java | 1 + .../hadoop/hdfs/server/namenode/FSImage.java | 44 +- .../hdfs/server/namenode/FSImageFormat.java | 45 +- .../server/namenode/FSImageFormatPBINode.java | 26 +- .../server/namenode/FSImageFormatProtobuf.java | 42 +- .../server/namenode/FSImageSerialization.java | 2 +- .../hdfs/server/namenode/FSImageUtil.java | 4 +- .../hdfs/server/namenode/FSNDNCacheOp.java | 124 + .../hdfs/server/namenode/FSNamesystem.java | 3286 ++++++---------- .../server/namenode/FSPermissionChecker.java | 116 +- .../server/namenode/FileJournalManager.java | 2 +- .../hadoop/hdfs/server/namenode/INode.java | 21 +- .../hdfs/server/namenode/INodeAttributes.java | 9 +- .../hdfs/server/namenode/INodeDirectory.java | 19 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 190 +- .../namenode/INodeWithAdditionalFields.java | 5 +- .../hdfs/server/namenode/INodesInPath.java | 370 +- .../hdfs/server/namenode/ImageServlet.java | 18 +- .../namenode/InotifyFSEditLogOpTranslator.java | 98 +- .../hdfs/server/namenode/LeaseManager.java | 86 +- .../hadoop/hdfs/server/namenode/NNConf.java | 104 - .../hadoop/hdfs/server/namenode/NNStorage.java | 25 +- .../hadoop/hdfs/server/namenode/NameNode.java | 18 +- .../server/namenode/NameNodeLayoutVersion.java | 5 +- .../hdfs/server/namenode/NameNodeMXBean.java | 11 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 387 +- .../server/namenode/NameNodeStatusMXBean.java | 7 + .../hdfs/server/namenode/NamenodeFsck.java | 119 +- .../namenode/SequentialBlockIdGenerator.java | 66 - .../hdfs/server/namenode/TransferFsImage.java | 22 + .../server/namenode/XAttrPermissionFilter.java | 2 +- .../hdfs/server/namenode/ha/EditLogTailer.java | 12 +- .../hadoop/hdfs/server/namenode/ha/HAState.java | 17 +- .../server/namenode/ha/StandbyCheckpointer.java | 12 +- .../namenode/metrics/FSNamesystemMBean.java | 13 + .../namenode/metrics/NameNodeMetrics.java | 5 + .../snapshot/AbstractINodeDiffList.java | 15 +- .../snapshot/DirectoryWithSnapshotFeature.java | 5 + .../snapshot/FSImageFormatPBSnapshot.java | 36 +- .../hdfs/server/namenode/snapshot/FileDiff.java | 42 +- .../server/namenode/snapshot/FileDiffList.java | 98 + .../snapshot/FileWithSnapshotFeature.java | 63 +- .../hdfs/server/namenode/snapshot/Snapshot.java | 10 +- .../namenode/snapshot/SnapshotManager.java | 50 +- .../server/namenode/top/TopAuditLogger.java | 82 + .../hdfs/server/namenode/top/TopConf.java | 64 + .../server/namenode/top/metrics/TopMetrics.java | 131 + .../namenode/top/window/RollingWindow.java | 189 + .../top/window/RollingWindowManager.java | 346 ++ .../web/resources/NamenodeWebHdfsMethods.java | 73 +- .../server/protocol/BlockRecoveryCommand.java | 20 + .../server/protocol/DatanodeRegistration.java | 1 + .../hdfs/server/protocol/DatanodeStorage.java | 4 +- .../server/protocol/InterDatanodeProtocol.java | 3 +- .../hdfs/server/protocol/ServerCommand.java | 8 + .../hdfs/shortcircuit/ShortCircuitCache.java | 23 +- .../hdfs/shortcircuit/ShortCircuitReplica.java | 12 +- .../apache/hadoop/hdfs/tools/AdminHelper.java | 192 + .../apache/hadoop/hdfs/tools/CacheAdmin.java | 264 +- .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 130 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 117 +- .../apache/hadoop/hdfs/tools/DFSHAAdmin.java | 2 +- .../org/apache/hadoop/hdfs/tools/DFSck.java | 21 +- .../apache/hadoop/hdfs/tools/DebugAdmin.java | 361 ++ .../hadoop/hdfs/tools/GetStoragePolicies.java | 65 - .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 231 ++ .../OfflineEditsXmlLoader.java | 4 +- .../DelimitedImageVisitor.java | 2 +- .../offlineImageViewer/FSImageHandler.java | 166 +- .../tools/offlineImageViewer/FSImageLoader.java | 41 +- .../FileDistributionCalculator.java | 15 +- .../FileDistributionVisitor.java | 4 +- .../offlineImageViewer/LsImageVisitor.java | 6 +- .../OfflineImageViewerPB.java | 46 +- .../offlineImageViewer/PBImageXmlWriter.java | 19 +- .../offlineImageViewer/WebImageViewer.java | 109 +- .../offlineImageViewer/XmlImageVisitor.java | 4 +- .../hadoop/hdfs/util/ByteArrayManager.java | 418 ++ .../hadoop/hdfs/util/ChunkedArrayList.java | 171 - .../hadoop/hdfs/util/ReferenceCountMap.java | 118 + .../org/apache/hadoop/hdfs/util/XMLUtils.java | 79 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 17 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 3 +- .../hadoop/hdfs/web/resources/IntegerParam.java | 3 +- .../hadoop/hdfs/web/resources/LongParam.java | 3 +- .../hadoop/hdfs/web/resources/ShortParam.java | 3 +- .../hdfs/web/resources/TokenKindParam.java | 42 + .../hdfs/web/resources/TokenServiceParam.java | 44 + .../hadoop/hdfs/web/resources/UserProvider.java | 3 +- .../src/main/native/libhdfs/native_mini_dfs.c | 52 +- .../src/main/native/libhdfs/native_mini_dfs.h | 17 +- .../native/libhdfs/test/test_libhdfs_zerocopy.c | 31 + .../src/main/proto/ClientDatanodeProtocol.proto | 10 + .../src/main/proto/ClientNamenodeProtocol.proto | 13 + .../src/main/proto/InterDatanodeProtocol.proto | 2 + .../hadoop-hdfs/src/main/proto/acl.proto | 1 + .../src/main/proto/datatransfer.proto | 3 +- .../hadoop-hdfs/src/main/proto/fsimage.proto | 1 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 16 +- .../hadoop-hdfs/src/main/proto/inotify.proto | 11 +- .../resources/blockStoragePolicy-default.xml | 118 - .../src/main/resources/hdfs-default.xml | 76 +- .../src/main/webapps/hdfs/dfshealth.html | 10 +- .../src/main/webapps/hdfs/dfshealth.js | 17 + .../src/site/apt/ArchivalStorage.apt.vm | 209 +- .../src/site/apt/HDFSCommands.apt.vm | 64 +- .../src/site/apt/HdfsNfsGateway.apt.vm | 6 +- .../src/site/apt/HdfsUserGuide.apt.vm | 20 + .../src/site/apt/TransparentEncryption.apt.vm | 126 +- .../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 48 +- .../src/site/xdoc/HdfsRollingUpgrade.xml | 77 +- .../hadoop/fs/shell/TestHdfsTextCommand.java | 3 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 44 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 48 +- .../hadoop/hdfs/TestBlockReaderFactory.java | 3 + .../hadoop/hdfs/TestBlockStoragePolicy.java | 56 +- .../hdfs/TestDFSInotifyEventInputStream.java | 211 +- .../apache/hadoop/hdfs/TestDFSPermission.java | 18 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 45 +- .../hadoop/hdfs/TestDataTransferKeepalive.java | 8 +- .../hadoop/hdfs/TestEncryptedTransfer.java | 133 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 59 +- .../hadoop/hdfs/TestEncryptionZonesWithHA.java | 2 +- .../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 39 + .../org/apache/hadoop/hdfs/TestFetchImage.java | 9 +- .../org/apache/hadoop/hdfs/TestFileAppend3.java | 83 + .../hadoop/hdfs/TestFileAppendRestart.java | 2 +- .../org/apache/hadoop/hdfs/TestFileStatus.java | 27 +- .../org/apache/hadoop/hdfs/TestGetBlocks.java | 4 +- .../java/org/apache/hadoop/hdfs/TestHFlush.java | 10 +- .../hadoop/hdfs/TestMissingBlocksAlert.java | 19 +- .../hadoop/hdfs/TestMultiThreadedHflush.java | 11 +- .../apache/hadoop/hdfs/TestRenameWhileOpen.java | 2 +- .../org/apache/hadoop/hdfs/TestReplication.java | 32 +- .../hadoop/hdfs/TestReservedRawPaths.java | 2 +- .../apache/hadoop/hdfs/TestRollingUpgrade.java | 8 +- .../hadoop/hdfs/TestStoragePolicyCommands.java | 88 - .../hadoop/hdfs/protocol/TestLayoutVersion.java | 15 + .../datatransfer/TestPacketReceiver.java | 5 +- .../datatransfer/sasl/TestSaslDataTransfer.java | 28 +- .../hadoop/hdfs/qjournal/TestNNWithQJM.java | 2 +- .../hdfs/qjournal/TestSecureNNWithQJM.java | 216 ++ .../hdfs/qjournal/server/TestJournalNode.java | 9 +- .../hdfs/server/balancer/TestBalancer.java | 88 +- .../blockmanagement/BlockManagerTestUtil.java | 29 +- .../server/blockmanagement/TestBlockInfo.java | 10 +- .../TestBlockInfoUnderConstruction.java | 2 +- .../blockmanagement/TestBlockManager.java | 21 +- .../TestComputeInvalidateWork.java | 167 +- .../blockmanagement/TestDatanodeDescriptor.java | 12 +- .../blockmanagement/TestHeartbeatHandling.java | 6 +- .../TestNameNodePrunesMissingStorages.java | 121 + .../TestPendingInvalidateBlock.java | 11 + .../blockmanagement/TestReplicationPolicy.java | 78 +- .../TestReplicationPolicyConsiderLoad.java | 6 +- .../blockmanagement/TestSequentialBlockId.java | 197 + .../TestUnderReplicatedBlockQueues.java | 19 +- .../TestUnderReplicatedBlocks.java | 77 + .../server/datanode/SimulatedFSDataset.java | 73 +- .../hdfs/server/datanode/TestBlockRecovery.java | 53 +- .../server/datanode/TestBlockReplacement.java | 101 +- .../hdfs/server/datanode/TestDataNodeExit.java | 16 + .../datanode/TestDataNodeHotSwapVolumes.java | 199 +- .../server/datanode/TestDataNodeMetrics.java | 60 + .../datanode/TestDataNodeVolumeFailure.java | 54 +- .../hdfs/server/datanode/TestDataStorage.java | 26 +- .../datanode/TestHdfsServerConstants.java | 17 +- .../datanode/TestIncrementalBlockReports.java | 2 +- .../server/datanode/TestTriggerBlockReport.java | 131 + .../fsdataset/impl/LazyPersistTestCase.java | 433 +++ .../fsdataset/impl/TestFsDatasetImpl.java | 91 +- .../impl/TestInterDatanodeProtocol.java | 9 +- .../fsdataset/impl/TestLazyPersistFiles.java | 357 +- .../fsdataset/impl/TestScrLazyPersistFiles.java | 356 +- .../resources/TestDatanodeWebHdfsMethods.java | 54 - .../web/webhdfs/TestParameterParser.java | 55 + .../hadoop/hdfs/server/mover/TestMover.java | 48 +- .../hdfs/server/mover/TestStorageMover.java | 50 +- .../hdfs/server/namenode/FSAclBaseTest.java | 337 +- .../hdfs/server/namenode/NameNodeAdapter.java | 10 +- .../hdfs/server/namenode/TestAuditLogger.java | 113 +- .../hdfs/server/namenode/TestCheckpoint.java | 33 +- .../TestCommitBlockSynchronization.java | 9 +- .../hdfs/server/namenode/TestDeadDatanode.java | 32 +- .../namenode/TestDecommissioningStatus.java | 52 +- .../TestDefaultBlockPlacementPolicy.java | 121 + .../hdfs/server/namenode/TestEditLog.java | 78 +- .../hdfs/server/namenode/TestEditLogRace.java | 43 +- .../hdfs/server/namenode/TestFSDirectory.java | 47 +- .../namenode/TestFSImageWithSnapshot.java | 2 +- .../server/namenode/TestFSNamesystemMBean.java | 69 +- .../namenode/TestFSPermissionChecker.java | 20 +- .../server/namenode/TestFileContextAcl.java | 6 +- .../hdfs/server/namenode/TestFileTruncate.java | 740 ++++ .../hdfs/server/namenode/TestFsLimits.java | 4 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 259 +- .../hdfs/server/namenode/TestINodeFile.java | 20 +- .../hdfs/server/namenode/TestLeaseManager.java | 27 + .../hdfs/server/namenode/TestMetaSave.java | 23 +- .../hdfs/server/namenode/TestNameNodeAcl.java | 6 +- .../server/namenode/TestNameNodeMXBean.java | 244 +- .../namenode/TestNameNodeOptionParsing.java | 45 + .../server/namenode/TestNameNodeRecovery.java | 3 +- .../namenode/TestNamenodeCapacityReport.java | 72 +- .../server/namenode/TestNamenodeRetryCache.java | 133 +- .../hdfs/server/namenode/TestSaveNamespace.java | 11 +- .../server/namenode/TestSequentialBlockId.java | 209 - .../server/namenode/TestSnapshotPathINodes.java | 202 +- .../server/namenode/ha/HAStressTestHarness.java | 4 +- .../hdfs/server/namenode/ha/TestDNFencing.java | 68 +- .../ha/TestDNFencingWithReplication.java | 4 + .../namenode/ha/TestDelegationTokensWithHA.java | 2 +- .../namenode/ha/TestEditLogsDuringFailover.java | 2 +- .../hdfs/server/namenode/ha/TestHAMetrics.java | 13 + .../namenode/ha/TestRetryCacheWithHA.java | 7 +- .../namenode/ha/TestStandbyCheckpoints.java | 23 +- .../namenode/metrics/TestNameNodeMetrics.java | 3 + .../namenode/snapshot/SnapshotTestHelper.java | 35 +- .../namenode/snapshot/TestAclWithSnapshot.java | 189 +- .../snapshot/TestOpenFilesWithSnapshot.java | 4 +- .../snapshot/TestRenameWithSnapshots.java | 8 +- .../server/namenode/snapshot/TestSnapshot.java | 8 +- .../snapshot/TestSnapshotFileLength.java | 42 +- .../namenode/snapshot/TestSnapshotManager.java | 14 +- .../snapshot/TestSnapshotReplication.java | 11 +- .../namenode/top/window/TestRollingWindow.java | 84 + .../top/window/TestRollingWindowManager.java | 102 + .../shortcircuit/TestShortCircuitCache.java | 1 + .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 3 +- .../hadoop/hdfs/tools/TestDebugAdmin.java | 118 + .../hdfs/tools/TestStoragePolicyCommands.java | 93 + .../TestOfflineImageViewer.java | 44 +- .../TestOfflineImageViewerForAcl.java | 5 +- .../hadoop/hdfs/util/TestByteArrayManager.java | 646 ++++ .../hadoop/hdfs/util/TestChunkedArrayList.java | 93 - .../apache/hadoop/hdfs/util/TestXMLUtils.java | 31 +- .../hdfs/web/TestFSMainOperationsWebHdfs.java | 2 - .../apache/hadoop/hdfs/web/TestWebHDFSAcl.java | 6 +- .../hdfs/web/TestWebHdfsFileSystemContract.java | 4 +- .../hadoop/hdfs/web/TestWebHdfsTokens.java | 70 +- .../apache/hadoop/tracing/TestTraceAdmin.java | 17 +- .../org/apache/hadoop/tracing/TestTracing.java | 10 +- .../hadoop-hdfs/src/test/resources/editsStored | Bin 5065 -> 5586 bytes .../src/test/resources/editsStored.xml | 408 +- .../test/resources/hadoop-24-datanode-dir.tgz | Bin 637608 -> 657548 bytes .../src/test/resources/testAclCLI.xml | 53 + .../src/test/resources/testCryptoConf.xml | 30 +- .../src/test/resources/testHDFSConf.xml | 437 ++- hadoop-mapreduce-project/CHANGES.txt | 133 +- hadoop-mapreduce-project/bin/mapred | 22 +- hadoop-mapreduce-project/bin/mapred-config.sh | 2 +- hadoop-mapreduce-project/bin/mapred.cmd | 11 +- hadoop-mapreduce-project/conf/mapred-env.cmd | 2 +- hadoop-mapreduce-project/conf/mapred-env.sh | 21 +- .../hadoop-mapreduce-client-app/pom.xml | 6 + .../apache/hadoop/mapred/MapReduceChildJVM.java | 35 +- .../org/apache/hadoop/mapred/WrappedJvmID.java | 4 +- .../org/apache/hadoop/mapred/YarnChild.java | 4 +- .../mapreduce/jobhistory/JobHistoryEvent.java | 7 +- .../jobhistory/JobHistoryEventHandler.java | 356 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 6 +- .../v2/app/client/MRClientService.java | 24 +- .../v2/app/job/impl/TaskAttemptImpl.java | 12 +- .../v2/app/local/LocalContainerAllocator.java | 47 +- .../mapreduce/v2/app/rm/RMCommunicator.java | 3 +- .../v2/app/rm/RMContainerAllocator.java | 46 +- .../v2/app/rm/RMContainerRequestor.java | 23 +- .../authorize/ClientHSPolicyProvider.java | 17 +- .../jobhistory/TestJobHistoryEventHandler.java | 251 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 6 +- .../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 11 +- .../hadoop/mapreduce/v2/app/MockJobs.java | 6 +- .../v2/app/TestCheckpointPreemptionPolicy.java | 4 +- .../v2/app/TestKillAMPreemptionPolicy.java | 2 +- .../mapreduce/v2/app/TestMRAppMaster.java | 2 +- .../mapreduce/v2/app/TestMRClientService.java | 16 + .../mapreduce/v2/app/TestStagingCleanup.java | 2 +- .../v2/app/job/impl/TestMapReduceChildJVM.java | 99 +- .../v2/app/job/impl/TestTaskAttempt.java | 22 +- .../v2/app/launcher/TestContainerLauncher.java | 8 +- .../app/launcher/TestContainerLauncherImpl.java | 30 +- .../v2/app/rm/TestRMContainerAllocator.java | 42 +- .../hadoop-mapreduce-client-common/pom.xml | 1 + .../mapreduce/v2/api/HSAdminProtocol.java | 34 + .../v2/api/HSAdminRefreshProtocol.java | 59 + .../v2/api/HSAdminRefreshProtocolPB.java | 33 + .../mapreduce/v2/jobhistory/JHAdminConfig.java | 4 +- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 94 +- .../src/main/proto/HSAdminRefreshProtocol.proto | 100 + .../hadoop-mapreduce-client-core/.gitattributes | 1 + .../hadoop-mapreduce-client-core/pom.xml | 1 + .../apache/hadoop/mapred/FileInputFormat.java | 12 +- .../apache/hadoop/mapred/FileOutputFormat.java | 4 +- .../java/org/apache/hadoop/mapred/JVMId.java | 83 +- .../java/org/apache/hadoop/mapred/JobConf.java | 3 +- .../apache/hadoop/mapred/LineRecordReader.java | 1 + .../java/org/apache/hadoop/mapred/MapTask.java | 5 +- .../java/org/apache/hadoop/mapred/Merger.java | 23 +- .../java/org/apache/hadoop/mapred/TaskLog.java | 4 + .../org/apache/hadoop/mapreduce/Cluster.java | 16 +- .../apache/hadoop/mapreduce/CryptoUtils.java | 3 +- .../hadoop/mapreduce/JobSubmissionFiles.java | 8 + .../apache/hadoop/mapreduce/JobSubmitter.java | 125 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 35 +- .../mapreduce/counters/AbstractCounters.java | 4 + .../hadoop/mapreduce/counters/Limits.java | 12 + .../mapreduce/jobhistory/HistoryViewer.java | 18 +- .../mapreduce/lib/input/FileInputFormat.java | 12 +- .../mapreduce/lib/input/LineRecordReader.java | 1 + .../hadoop/mapreduce/task/reduce/Fetcher.java | 41 +- .../mapreduce/task/reduce/LocalFetcher.java | 2 +- .../mapreduce/task/reduce/OnDiskMapOutput.java | 9 +- .../task/reduce/ShuffleSchedulerImpl.java | 107 +- .../hadoop/mapreduce/util/ConfigUtil.java | 2 - .../hadoop/mapreduce/util/ProcessTree.java | 4 +- .../src/main/resources/mapred-default.xml | 84 +- .../src/site/apt/EncryptedShuffle.apt.vm | 8 +- .../src/site/apt/MapredCommands.apt.vm | 5 +- .../hadoop/mapred/TestLineRecordReader.java | 58 + .../hadoop/mapreduce/lib/db/TestDbClasses.java | 8 +- .../lib/input/TestLineRecordReader.java | 38 + .../mapreduce/task/reduce/TestFetcher.java | 72 +- .../mapreduce/task/reduce/TestMerger.java | 82 +- .../task/reduce/TestShuffleScheduler.java | 153 + .../hadoop-mapreduce-client-hs/pom.xml | 30 - .../hadoop/mapreduce/v2/hs/CompletedJob.java | 29 +- .../hadoop/mapreduce/v2/hs/HSProxies.java | 4 +- .../mapreduce/v2/hs/HistoryClientService.java | 22 +- .../mapreduce/v2/hs/HistoryFileManager.java | 2 +- .../hadoop/mapreduce/v2/hs/client/HSAdmin.java | 18 +- .../v2/hs/protocol/HSAdminProtocol.java | 38 - .../v2/hs/protocol/HSAdminRefreshProtocol.java | 63 - ...inRefreshProtocolClientSideTranslatorPB.java | 3 +- .../hs/protocolPB/HSAdminRefreshProtocolPB.java | 33 - ...inRefreshProtocolServerSideTranslatorPB.java | 3 +- .../mapreduce/v2/hs/server/HSAdminServer.java | 14 +- .../src/main/proto/HSAdminRefreshProtocol.proto | 100 - .../mapreduce/v2/hs/TestJobHistoryEntities.java | 30 +- .../mapreduce/v2/hs/TestJobHistoryServer.java | 19 + .../v2/hs/server/TestHSAdminServer.java | 20 + .../mapreduce/v2/hs/webapp/TestBlocks.java | 2 +- .../resources/job_1329348432655_0001_conf.xml | 1 - ...33-2-0-SUCCEEDED-default-1416424779349.jhist | 29 + .../hadoop-mapreduce-client-jobclient/pom.xml | 6 + .../hadoop/mapred/ResourceMgrDelegate.java | 16 +- .../org/apache/hadoop/mapred/YARNRunner.java | 105 +- .../fs/loadGenerator/LoadGeneratorMR.java | 483 +++ .../hadoop/mapred/TestClientRedirect.java | 23 +- .../hadoop/mapred/TestJavaSerialization.java | 36 +- .../mapred/TestMRTimelineEventHandling.java | 169 + .../hadoop/mapred/TestMiniMRChildTask.java | 31 +- .../apache/hadoop/mapred/TestYARNRunner.java | 43 +- .../mapred/pipes/TestPipeApplication.java | 2 +- .../apache/hadoop/mapreduce/TestCounters.java | 39 +- .../apache/hadoop/mapreduce/TestMapReduce.java | 12 +- .../lib/input/TestCombineFileInputFormat.java | 4 - .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 7 +- .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 131 +- .../v2/TestMRJobsWithHistoryService.java | 2 +- .../apache/hadoop/mapreduce/v2/TestUberAM.java | 40 +- .../apache/hadoop/test/MapredTestDriver.java | 12 + .../hadoop-mapreduce-client-nativetask/pom.xml | 2 - .../src/main/native/src/codec/Lz4Codec.cc | 2 +- .../src/main/native/test/TestCompressions.cc | 2 +- .../hadoop/mapred/nativetask/kvtest/KVJob.java | 8 +- .../apache/hadoop/mapred/ShuffleHandler.java | 4 +- .../org/apache/hadoop/examples/pi/Parser.java | 5 +- .../apache/hadoop/examples/pi/math/Bellard.java | 9 +- .../hadoop/examples/terasort/TeraGen.java | 4 - .../examples/terasort/TeraOutputFormat.java | 27 +- .../hadoop/examples/terasort/TestTeraSort.java | 42 +- hadoop-maven-plugins/pom.xml | 2 +- .../apache/hadoop/maven/plugin/util/Exec.java | 4 +- .../hadoop/maven/plugin/util/FileSetUtils.java | 2 +- hadoop-minicluster/pom.xml | 2 +- hadoop-project/pom.xml | 56 +- hadoop-project/src/site/site.xml | 17 +- hadoop-tools/hadoop-ant/pom.xml | 56 + .../java/org/apache/hadoop/ant/DfsTask.java | 210 ++ .../ant/condition/DfsBaseConditional.java | 68 + .../apache/hadoop/ant/condition/DfsExists.java | 24 + .../apache/hadoop/ant/condition/DfsIsDir.java | 24 + .../apache/hadoop/ant/condition/DfsZeroLen.java | 24 + .../resources/org/apache/hadoop/ant/antlib.xml | 29 + .../org/apache/hadoop/tools/HadoopArchives.java | 17 +- .../apache/hadoop/tools/TestHadoopArchives.java | 77 +- hadoop-tools/hadoop-aws/pom.xml | 9 +- .../org/apache/hadoop/fs/s3/S3Credentials.java | 4 +- .../fs/s3a/BasicAWSCredentialsProvider.java | 8 +- .../org/apache/hadoop/fs/s3a/Constants.java | 24 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 242 +- .../apache/hadoop/fs/s3a/S3AInputStream.java | 38 +- .../apache/hadoop/fs/s3a/S3AOutputStream.java | 31 +- .../src/site/markdown/tools/hadoop-aws/index.md | 417 ++ .../fs/contract/s3a/TestS3AContractRename.java | 13 +- .../fs/s3/S3FileSystemContractBaseTest.java | 11 +- .../fs/s3a/S3AFileSystemContractBaseTest.java | 327 -- .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 51 + .../fs/s3a/TestS3AFileSystemContract.java | 105 + .../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 89 + .../fs/s3a/scale/TestS3ADeleteManyFiles.java | 131 + .../NativeS3FileSystemContractBaseTest.java | 11 +- .../TestJets3tNativeFileSystemStore.java | 3 + .../src/test/resources/contract/s3a.xml | 5 + .../hadoop-aws/src/test/resources/core-site.xml | 51 + hadoop-tools/hadoop-azure/README.txt | 118 - .../dev-support/findbugs-exclude.xml | 31 + hadoop-tools/hadoop-azure/pom.xml | 17 +- .../hadoop-azure/src/config/checkstyle.xml | 7 +- .../hadoop/fs/azure/AzureLinkedStack.java | 217 ++ .../fs/azure/AzureNativeFileSystemStore.java | 740 ++-- .../hadoop/fs/azure/NativeAzureFileSystem.java | 1358 +++++-- .../hadoop/fs/azure/NativeFileSystemStore.java | 22 +- .../hadoop/fs/azure/PageBlobFormatHelpers.java | 58 + .../hadoop/fs/azure/PageBlobInputStream.java | 455 +++ .../hadoop/fs/azure/PageBlobOutputStream.java | 584 +++ .../apache/hadoop/fs/azure/PartialListing.java | 2 +- .../hadoop/fs/azure/SelfRenewingLease.java | 203 + .../fs/azure/SelfThrottlingIntercept.java | 4 +- .../fs/azure/ShellDecryptionKeyProvider.java | 3 +- .../hadoop/fs/azure/SimpleKeyProvider.java | 3 +- .../hadoop/fs/azure/StorageInterface.java | 280 +- .../hadoop/fs/azure/StorageInterfaceImpl.java | 184 +- .../fs/azure/SyncableDataOutputStream.java | 56 + .../java/org/apache/hadoop/fs/azure/Wasb.java | 1 - .../metrics/AzureFileSystemInstrumentation.java | 5 +- .../metrics/ResponseReceivedMetricUpdater.java | 7 +- .../services/org.apache.hadoop.fs.FileSystem | 17 + .../hadoop-azure/src/site/markdown/index.md | 243 ++ .../fs/azure/AzureBlobStorageTestAccount.java | 155 +- .../hadoop/fs/azure/InMemoryBlockBlobStore.java | 70 +- .../hadoop/fs/azure/MockStorageInterface.java | 233 +- .../fs/azure/NativeAzureFileSystemBaseTest.java | 1016 ++++- .../hadoop/fs/azure/RunningLiveWasbTests.txt | 22 + .../azure/TestAzureConcurrentOutOfBandIo.java | 75 +- .../TestAzureFileSystemErrorConditions.java | 57 +- .../hadoop/fs/azure/TestBlobDataValidation.java | 12 +- .../hadoop/fs/azure/TestBlobMetadata.java | 35 +- .../fs/azure/TestBlobTypeSpeedDifference.java | 160 + .../fs/azure/TestNativeAzureFSPageBlobLive.java | 43 + .../TestNativeAzureFileSystemConcurrency.java | 5 +- .../TestNativeAzureFileSystemContractLive.java | 26 + ...TestNativeAzureFileSystemContractMocked.java | 25 + .../TestNativeAzureFileSystemFileNameCheck.java | 11 +- .../fs/azure/TestNativeAzureFileSystemLive.java | 75 + .../azure/TestNativeAzureFileSystemMocked.java | 35 + ...stNativeAzureFileSystemOperationsMocked.java | 37 +- .../TestNativeAzureFileSystemUploadLogic.java | 186 + .../azure/TestOutOfBandAzureBlobOperations.java | 17 +- .../TestOutOfBandAzureBlobOperationsLive.java | 21 + .../TestReadAndSeekPageBlobAfterWrite.java | 365 ++ .../apache/hadoop/fs/azure/TestWasbFsck.java | 33 + .../fs/azure/TestWasbUriAndConfiguration.java | 9 +- .../fs/azure/metrics/AzureMetricsTestUtil.java | 1 + .../TestAzureFileSystemInstrumentation.java | 53 +- .../metrics/TestBandwidthGaugeUpdater.java | 47 - .../TestNativeAzureFileSystemMetricsSystem.java | 11 + .../src/test/resources/azure-test.xml | 12 - .../hadoop/tools/FileBasedCopyListing.java | 4 +- .../hadoop/tools/util/ThrottledInputStream.java | 2 +- .../java/org/apache/hadoop/tools/DistCh.java | 23 +- .../java/org/apache/hadoop/tools/DistCpV1.java | 70 +- .../java/org/apache/hadoop/tools/DistTool.java | 8 +- .../java/org/apache/hadoop/tools/Logalyzer.java | 9 +- .../dev-support/findbugs-exclude.xml | 7 + .../gridmix/CompressionEmulationUtil.java | 8 +- .../gridmix/DistributedCacheEmulator.java | 10 +- .../mapred/gridmix/GenerateDistCacheData.java | 6 +- .../apache/hadoop/mapred/gridmix/Gridmix.java | 36 +- .../hadoop/mapred/gridmix/PseudoLocalFs.java | 2 +- .../hadoop/mapred/gridmix/Statistics.java | 26 +- hadoop-tools/hadoop-openstack/pom.xml | 17 +- .../snative/SwiftNativeFileSystemStore.java | 7 +- .../hadoop/fs/swift/util/SwiftTestUtils.java | 4 +- .../apache/hadoop/tools/rumen/JobBuilder.java | 11 +- hadoop-tools/hadoop-sls/pom.xml | 2 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 + .../yarn/sls/scheduler/RMNodeWrapper.java | 6 + .../sls/scheduler/ResourceSchedulerWrapper.java | 6 + .../yarn/sls/nodemanager/TestNMSimulator.java | 10 + .../apache/hadoop/streaming/Environment.java | 4 +- .../org/apache/hadoop/streaming/PipeMapRed.java | 4 +- .../streaming/TestStreamXmlMultipleRecords.java | 4 +- .../java/org/apache/hadoop/streaming/TrApp.java | 3 + hadoop-tools/hadoop-tools-dist/pom.xml | 6 + hadoop-tools/pom.xml | 1 + hadoop-yarn-project/CHANGES.txt | 674 +++- .../hadoop-yarn/bin/start-yarn.sh | 6 +- .../hadoop-yarn/bin/stop-yarn.sh | 12 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 62 +- .../hadoop-yarn/bin/yarn-config.sh | 2 +- .../hadoop-yarn/bin/yarn-daemons.sh | 6 +- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 16 +- .../hadoop-yarn/conf/yarn-env.cmd | 2 +- .../hadoop-yarn/conf/yarn-env.sh | 59 +- .../dev-support/findbugs-exclude.xml | 86 + .../hadoop-yarn/hadoop-yarn-api/pom.xml | 2 + .../yarn/api/ApplicationClientProtocol.java | 42 +- .../yarn/api/ApplicationMasterProtocol.java | 3 + .../hadoop/yarn/api/ClientSCMProtocol.java | 90 + .../hadoop/yarn/api/ClientSCMProtocolPB.java | 28 + .../api/protocolrecords/AllocateResponse.java | 2 +- .../GetClusterNodeLabelsRequest.java | 31 + .../GetClusterNodeLabelsResponse.java | 44 + .../GetNodesToLabelsRequest.java | 27 + .../GetNodesToLabelsResponse.java | 45 + .../ReleaseSharedCacheResourceRequest.java | 67 + .../ReleaseSharedCacheResourceResponse.java | 37 + .../UseSharedCacheResourceRequest.java | 70 + .../UseSharedCacheResourceResponse.java | 55 + .../hadoop/yarn/api/records/AMCommand.java | 26 +- .../records/ApplicationSubmissionContext.java | 109 +- .../hadoop/yarn/api/records/ContainerId.java | 21 +- .../hadoop/yarn/api/records/LocalResource.java | 38 + .../yarn/api/records/LogAggregationContext.java | 28 +- .../hadoop/yarn/api/records/NodeReport.java | 27 +- .../hadoop/yarn/api/records/QueueInfo.java | 40 +- .../yarn/api/records/ResourceRequest.java | 51 + .../yarn/api/records/SerializedException.java | 18 + .../api/records/timeline/TimelineEntity.java | 49 +- .../api/records/timeline/TimelineEvent.java | 17 +- .../hadoop/yarn/conf/YarnConfiguration.java | 268 +- .../ApplicationAttemptNotFoundException.java | 6 +- .../ResourceManagerAdministrationProtocol.java | 24 + .../yarn/server/api/SCMAdminProtocol.java | 53 + .../yarn/server/api/SCMAdminProtocolPB.java | 31 + .../AddToClusterNodeLabelsRequest.java | 44 + .../AddToClusterNodeLabelsResponse.java | 31 + .../RemoveFromClusterNodeLabelsRequest.java | 45 + .../RemoveFromClusterNodeLabelsResponse.java | 31 + .../ReplaceLabelsOnNodeRequest.java | 47 + .../ReplaceLabelsOnNodeResponse.java | 31 + .../RunSharedCacheCleanerTaskRequest.java | 37 + .../RunSharedCacheCleanerTaskResponse.java | 58 + .../main/proto/applicationclient_protocol.proto | 3 +- .../src/main/proto/client_SCM_protocol.proto | 30 + .../main/proto/server/SCM_Admin_protocol.proto | 29 + ...esourcemanager_administration_protocol.proto | 3 + ..._server_resourcemanager_service_protos.proto | 21 + .../src/main/proto/yarn_protos.proto | 12 +- .../src/main/proto/yarn_service_protos.proto | 45 + .../distributedshell/ApplicationMaster.java | 142 +- .../applications/distributedshell/Client.java | 91 +- .../distributedshell/DSConstants.java | 5 + .../distributedshell/TestDistributedShell.java | 99 +- .../TestDistributedShellWithNodeLabels.java | 165 + .../UnmanagedAMLauncher.java | 11 +- .../org/apache/hadoop/yarn/client/SCMAdmin.java | 183 + .../hadoop/yarn/client/api/AMRMClient.java | 36 +- .../hadoop/yarn/client/api/YarnClient.java | 30 + .../api/async/impl/AMRMClientAsyncImpl.java | 40 +- .../yarn/client/api/impl/AMRMClientImpl.java | 39 +- .../impl/ContainerManagementProtocolProxy.java | 80 +- .../yarn/client/api/impl/YarnClientImpl.java | 38 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 109 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 31 +- .../apache/hadoop/yarn/client/cli/NodeCLI.java | 18 +- .../apache/hadoop/yarn/client/cli/QueueCLI.java | 155 + .../hadoop/yarn/client/cli/RMAdminCLI.java | 210 +- .../hadoop/yarn/client/ProtocolHATestBase.java | 72 +- .../TestApplicationMasterServiceOnHA.java | 92 - ...estApplicationMasterServiceProtocolOnHA.java | 118 + .../hadoop/yarn/client/TestRMAdminCLI.java | 371 -- .../hadoop/yarn/client/TestRMFailover.java | 8 +- .../api/async/impl/TestAMRMClientAsync.java | 31 +- .../api/async/impl/TestNMClientAsync.java | 8 +- .../yarn/client/api/impl/TestAHSClient.java | 16 +- .../yarn/client/api/impl/TestAMRMClient.java | 31 +- .../api/impl/TestAMRMClientOnRMRestart.java | 2 +- .../yarn/client/api/impl/TestNMClient.java | 10 +- .../yarn/client/api/impl/TestYarnClient.java | 36 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 184 +- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 551 +++ .../hadoop/yarn/client/cli/TestYarnCLI.java | 275 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 5 + .../hadoop/yarn/ContainerLogAppender.java | 18 +- .../yarn/ContainerRollingLogAppender.java | 11 +- .../ApplicationClientProtocolPBClientImpl.java | 61 +- .../client/ClientSCMProtocolPBClientImpl.java | 93 + .../ApplicationClientProtocolPBServiceImpl.java | 72 +- .../service/ClientSCMProtocolPBServiceImpl.java | 78 + .../pb/GetClusterNodeLabelsRequestPBImpl.java | 68 + .../pb/GetClusterNodeLabelsResponsePBImpl.java | 113 + .../impl/pb/GetNodesToLabelsRequestPBImpl.java | 69 + .../impl/pb/GetNodesToLabelsResponsePBImpl.java | 168 + ...ReleaseSharedCacheResourceRequestPBImpl.java | 122 + ...eleaseSharedCacheResourceResponsePBImpl.java | 53 + .../pb/UseSharedCacheResourceRequestPBImpl.java | 120 + .../UseSharedCacheResourceResponsePBImpl.java | 79 + .../pb/ApplicationSubmissionContextPBImpl.java | 71 +- .../api/records/impl/pb/ContainerIdPBImpl.java | 1 + .../records/impl/pb/LocalResourcePBImpl.java | 20 + .../impl/pb/LogAggregationContextPBImpl.java | 15 - .../api/records/impl/pb/NodeReportPBImpl.java | 31 +- .../api/records/impl/pb/PriorityPBImpl.java | 2 +- .../yarn/api/records/impl/pb/ProtoUtils.java | 3 +- .../api/records/impl/pb/QueueInfoPBImpl.java | 45 + .../records/impl/pb/ResourceRequestPBImpl.java | 19 + .../org/apache/hadoop/yarn/client/RMProxy.java | 2 + .../apache/hadoop/yarn/client/ServerProxy.java | 2 + .../hadoop/yarn/client/api/TimelineClient.java | 30 + .../client/api/impl/TimelineAuthenticator.java | 269 -- .../client/api/impl/TimelineClientImpl.java | 468 ++- .../hadoop/yarn/event/AsyncDispatcher.java | 22 +- .../AggregatedLogDeletionService.java | 93 +- .../logaggregation/AggregatedLogFormat.java | 156 +- .../logaggregation/LogAggregationUtils.java | 5 + .../yarn/logaggregation/LogCLIHelpers.java | 137 +- .../nodelabels/CommonNodeLabelsManager.java | 726 ++++ .../nodelabels/FileSystemNodeLabelsStore.java | 261 ++ .../hadoop/yarn/nodelabels/NodeLabel.java | 96 + .../hadoop/yarn/nodelabels/NodeLabelsStore.java | 66 + .../nodelabels/event/NodeLabelsStoreEvent.java | 28 + .../event/NodeLabelsStoreEventType.java | 25 + .../event/RemoveClusterNodeLabels.java | 34 + .../event/StoreNewClusterNodeLabels.java | 34 + .../event/UpdateNodeToLabelsMappingsEvent.java | 37 + .../client/ClientToAMTokenSecretManager.java | 25 +- .../TimelineDelegationTokenIdentifier.java | 52 +- .../client/YARNDelegationTokenIdentifier.java | 203 +- ...nagerAdministrationProtocolPBClientImpl.java | 59 +- .../pb/client/SCMAdminProtocolPBClientImpl.java | 73 + ...agerAdministrationProtocolPBServiceImpl.java | 64 + .../service/SCMAdminProtocolPBServiceImpl.java | 57 + .../pb/AddToClusterNodeLabelsRequestPBImpl.java | 114 + .../AddToClusterNodeLabelsResponsePBImpl.java | 69 + ...emoveFromClusterNodeLabelsRequestPBImpl.java | 115 + ...moveFromClusterNodeLabelsResponsePBImpl.java | 69 + .../pb/ReplaceLabelsOnNodeRequestPBImpl.java | 168 + .../pb/ReplaceLabelsOnNodeResponsePBImpl.java | 69 + .../RunSharedCacheCleanerTaskRequestPBImpl.java | 53 + ...RunSharedCacheCleanerTaskResponsePBImpl.java | 66 + .../server/security/ApplicationACLsManager.java | 7 + .../yarn/sharedcache/ChecksumSHA256Impl.java | 37 + .../yarn/sharedcache/SharedCacheChecksum.java | 43 + .../sharedcache/SharedCacheChecksumFactory.java | 84 + .../org/apache/hadoop/yarn/state/Graph.java | 17 +- .../yarn/state/VisualizeStateMachine.java | 2 +- .../apache/hadoop/yarn/util/ConverterUtils.java | 9 +- .../org/apache/hadoop/yarn/util/FSDownload.java | 35 +- .../util/LinuxResourceCalculatorPlugin.java | 19 +- .../yarn/util/ProcfsBasedProcessTree.java | 28 +- .../yarn/util/ResourceCalculatorPlugin.java | 12 - .../java/org/apache/hadoop/yarn/util/Times.java | 3 +- .../yarn/util/WindowsBasedProcessTree.java | 4 +- .../yarn/util/resource/ResourceCalculator.java | 5 - .../hadoop/yarn/webapp/YarnWebParams.java | 1 + .../yarn/webapp/log/AggregatedLogsBlock.java | 223 +- .../hadoop/yarn/webapp/util/WebAppUtils.java | 19 +- .../hadoop/yarn/webapp/view/JQueryUI.java | 3 +- .../main/proto/server/yarn_security_token.proto | 3 +- .../resources/webapps/static/yarn.dt.plugins.js | 27 +- .../src/main/resources/yarn-default.xml | 382 +- .../hadoop/yarn/TestContainerLaunchRPC.java | 2 +- .../hadoop/yarn/TestContainerLogAppender.java | 47 + .../java/org/apache/hadoop/yarn/TestRPC.java | 2 +- .../hadoop/yarn/api/TestAllocateResponse.java | 6 +- .../apache/hadoop/yarn/api/TestContainerId.java | 2 +- .../yarn/api/TestContainerResourceDecrease.java | 2 +- .../yarn/api/TestContainerResourceIncrease.java | 2 +- .../TestContainerResourceIncreaseRequest.java | 2 +- .../hadoop/yarn/api/TestPBImplRecords.java | 73 +- .../records/timeline/TestTimelineRecords.java | 128 +- .../api/impl/TestTimelineAuthenticator.java | 40 - .../client/api/impl/TestTimelineClient.java | 179 +- .../yarn/conf/TestYarnConfigurationFields.java | 106 + .../hadoop/yarn/event/DrainDispatcher.java | 50 +- .../TestAggregatedLogDeletionService.java | 192 +- .../logaggregation/TestAggregatedLogFormat.java | 66 +- .../logaggregation/TestAggregatedLogsBlock.java | 5 +- .../DummyCommonNodeLabelsManager.java | 86 + .../yarn/nodelabels/NodeLabelTestBase.java | 74 + .../nodelabels/TestCommonNodeLabelsManager.java | 322 ++ .../TestFileSystemNodeLabelsStore.java | 252 ++ .../yarn/security/TestYARNTokenIdentifier.java | 57 +- .../hadoop/yarn/util/TestConverterUtils.java | 14 + .../apache/hadoop/yarn/util/TestFSDownload.java | 3 +- .../hadoop/yarn/webapp/JerseyTestBase.java | 42 + .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 219 ++ .../apache/hadoop/registry/cli/RegistryCli.java | 444 +++ .../hadoop/registry/client/api/BindFlags.java | 41 + .../registry/client/api/RegistryConstants.java | 286 ++ .../registry/client/api/RegistryOperations.java | 182 + .../client/api/RegistryOperationsFactory.java | 131 + .../registry/client/api/package-info.java | 35 + .../registry/client/binding/JsonSerDeser.java | 309 ++ .../client/binding/RegistryPathUtils.java | 218 ++ .../client/binding/RegistryTypeUtils.java | 290 ++ .../registry/client/binding/RegistryUtils.java | 385 ++ .../registry/client/binding/package-info.java | 22 + .../AuthenticationFailedException.java | 39 + .../exceptions/InvalidPathnameException.java | 40 + .../exceptions/InvalidRecordException.java | 41 + .../NoChildrenForEphemeralsException.java | 48 + .../exceptions/NoPathPermissionsException.java | 45 + .../client/exceptions/NoRecordException.java | 45 + .../client/exceptions/RegistryIOException.java | 58 + .../client/exceptions/package-info.java | 33 + .../client/impl/RegistryOperationsClient.java | 55 + .../registry/client/impl/package-info.java | 26 + .../client/impl/zk/BindingInformation.java | 41 + .../registry/client/impl/zk/CuratorService.java | 772 ++++ .../client/impl/zk/RegistryBindingSource.java | 36 + .../impl/zk/RegistryInternalConstants.java | 81 + .../impl/zk/RegistryOperationsService.java | 163 + .../client/impl/zk/RegistrySecurity.java | 1000 +++++ .../registry/client/impl/zk/ZKPathDumper.java | 133 + .../client/impl/zk/ZookeeperConfigOptions.java | 119 + .../registry/client/impl/zk/package-info.java | 39 + .../registry/client/types/AddressTypes.java | 94 + .../hadoop/registry/client/types/Endpoint.java | 247 ++ .../registry/client/types/ProtocolTypes.java | 99 + .../client/types/RegistryPathStatus.java | 123 + .../registry/client/types/ServiceRecord.java | 249 ++ .../registry/client/types/package-info.java | 41 + .../client/types/yarn/PersistencePolicies.java | 50 + .../types/yarn/YarnRegistryAttributes.java | 31 + .../RMRegistryOperationsService.java | 246 ++ .../integration/SelectByYarnPersistence.java | 60 + .../server/integration/package-info.java | 23 + .../hadoop/registry/server/package-info.java | 27 + .../server/services/AddingCompositeService.java | 56 + .../services/DeleteCompletionCallback.java | 58 + .../server/services/MicroZookeeperService.java | 282 ++ .../services/MicroZookeeperServiceKeys.java | 69 + .../server/services/RegistryAdminService.java | 529 +++ .../registry/server/services/package-info.java | 40 + .../src/main/resources/.keep | 0 .../src/main/tla/yarnregistry.tla | 582 +++ .../hadoop/registry/AbstractRegistryTest.java | 123 + .../hadoop/registry/AbstractZKRegistryTest.java | 113 + .../hadoop/registry/RegistryTestHelper.java | 353 ++ .../client/binding/TestMarshalling.java | 143 + .../binding/TestRegistryOperationUtils.java | 56 + .../client/binding/TestRegistryPathUtils.java | 178 + .../client/impl/CuratorEventCatcher.java | 68 + .../client/impl/TestCuratorService.java | 249 ++ .../client/impl/TestMicroZookeeperService.java | 60 + .../integration/TestRegistryRMOperations.java | 369 ++ .../integration/TestYarnPolicySelector.java | 65 + .../operations/TestRegistryOperations.java | 331 ++ .../secure/AbstractSecureRegistryTest.java | 368 ++ .../registry/secure/KerberosConfiguration.java | 81 + .../secure/TestRegistrySecurityHelper.java | 211 ++ .../registry/secure/TestSecureLogins.java | 214 ++ .../secure/TestSecureRMRegistryOperations.java | 348 ++ .../registry/secure/TestSecureRegistry.java | 191 + .../src/test/resources/log4j.properties | 63 + .../pom.xml | 58 +- .../ApplicationHistoryClientService.java | 8 + ...pplicationHistoryManagerOnTimelineStore.java | 34 +- .../ApplicationHistoryServer.java | 13 +- .../webapp/AHSWebApp.java | 47 +- .../server/timeline/LeveldbTimelineStore.java | 500 ++- .../server/timeline/MemoryTimelineStore.java | 6 +- .../server/timeline/TimelineDataManager.java | 121 +- .../recovery/LeveldbTimelineStateStore.java | 420 +++ .../recovery/MemoryTimelineStateStore.java | 96 + .../timeline/recovery/TimelineStateStore.java | 193 + .../TimelineDelegationTokenIdentifierData.java | 63 + .../security/TimelineAuthenticationFilter.java | 32 +- ...TimelineAuthenticationFilterInitializer.java | 70 +- .../TimelineClientAuthenticationService.java | 236 -- ...lineDelegationTokenSecretManagerService.java | 240 +- .../yarn/server/timeline/util/LeveldbUtils.java | 141 + .../timeline/webapp/TimelineWebServices.java | 9 +- .../yarn_server_timelineserver_recovery.proto | 30 + .../ApplicationHistoryStoreTestUtils.java | 3 +- .../TestApplicationHistoryClientService.java | 111 +- ...pplicationHistoryManagerOnTimelineStore.java | 95 +- .../TestApplicationHistoryServer.java | 99 +- .../TestFileSystemApplicationHistoryStore.java | 13 +- .../TestMemoryApplicationHistoryStore.java | 13 +- .../webapp/TestAHSWebApp.java | 4 +- .../webapp/TestAHSWebServices.java | 177 +- .../timeline/TestLeveldbTimelineStore.java | 162 +- .../timeline/TestTimelineDataManager.java | 152 + .../server/timeline/TimelineStoreTestUtils.java | 16 + .../recovery/TestLeveldbTimelineStateStore.java | 213 ++ .../TestTimelineAuthenticationFilter.java | 333 ++ ...TimelineAuthenticationFilterInitializer.java | 81 + .../webapp/TestTimelineWebServices.java | 59 +- .../webapp/TestTimelineWebServicesWithSSL.java | 1 - .../hadoop-yarn-server-common/pom.xml | 1 + .../org/apache/hadoop/yarn/lib/ZKClient.java | 7 +- .../yarn/server/api/SCMUploaderProtocol.java | 83 + .../yarn/server/api/SCMUploaderProtocolPB.java | 28 + .../client/SCMUploaderProtocolPBClientImpl.java | 93 + .../SCMUploaderProtocolPBServiceImpl.java | 79 + .../protocolrecords/NodeHeartbeatResponse.java | 9 + .../SCMUploaderCanUploadRequest.java | 49 + .../SCMUploaderCanUploadResponse.java | 52 + .../SCMUploaderNotifyRequest.java | 67 + .../SCMUploaderNotifyResponse.java | 51 + .../impl/pb/NodeHeartbeatResponsePBImpl.java | 56 +- .../pb/SCMUploaderCanUploadRequestPBImpl.java | 78 + .../pb/SCMUploaderCanUploadResponsePBImpl.java | 75 + .../impl/pb/SCMUploaderNotifyRequestPBImpl.java | 93 + .../pb/SCMUploaderNotifyResponsePBImpl.java | 73 + .../security/http/RMAuthenticationFilter.java | 70 +- .../http/RMAuthenticationFilterInitializer.java | 22 +- .../sharedcache/SharedCacheStructureUtil.java | 81 - .../server/sharedcache/SharedCacheUtil.java | 91 + .../hadoop/yarn/server/utils/BuilderUtils.java | 31 +- .../hadoop/yarn/server/webapp/WebServices.java | 31 +- .../src/main/proto/SCMUploader.proto | 30 + .../yarn_server_common_service_protos.proto | 25 +- .../hadoop/yarn/TestYarnServerApiClasses.java | 6 +- .../protocolrecords/TestProtocolRecords.java | 46 +- .../TestRegisterNodeManagerRequest.java | 2 +- .../server/nodemanager/ContainerExecutor.java | 77 +- .../hadoop/yarn/server/nodemanager/Context.java | 4 + .../nodemanager/DefaultContainerExecutor.java | 152 +- .../server/nodemanager/DirectoryCollection.java | 221 +- .../nodemanager/DockerContainerExecutor.java | 794 ++++ .../nodemanager/LinuxContainerExecutor.java | 20 +- .../nodemanager/LocalDirsHandlerService.java | 176 +- .../nodemanager/NodeHealthCheckerService.java | 4 +- .../yarn/server/nodemanager/NodeManager.java | 17 + .../nodemanager/NodeStatusUpdaterImpl.java | 97 +- .../WindowsSecureContainerExecutor.java | 669 +++- .../containermanager/ContainerManagerImpl.java | 19 +- .../container/ContainerImpl.java | 244 +- .../launcher/ContainerLaunch.java | 75 +- .../launcher/RecoveredContainerLaunch.java | 20 +- .../localizer/ContainerLocalizer.java | 7 +- .../localizer/LocalResourceRequest.java | 11 + .../localizer/ResourceLocalizationService.java | 320 +- .../sharedcache/SharedCacheUploadEvent.java | 58 + .../sharedcache/SharedCacheUploadEventType.java | 28 + .../sharedcache/SharedCacheUploadService.java | 126 + .../sharedcache/SharedCacheUploader.java | 289 ++ .../logaggregation/AppLogAggregatorImpl.java | 219 +- .../logaggregation/LogAggregationService.java | 18 +- .../loghandler/NonAggregatingLogHandler.java | 47 +- .../nodemanager/metrics/NodeManagerMetrics.java | 44 +- .../recovery/NMLeveldbStateStoreService.java | 28 +- .../util/CgroupsLCEResourcesHandler.java | 42 +- .../nodemanager/util/ProcessIdFileReader.java | 12 +- .../nodemanager/webapp/ContainerLogsPage.java | 4 +- .../nodemanager/webapp/ContainerLogsUtils.java | 10 +- .../impl/container-executor.c | 65 +- .../impl/container-executor.h | 6 + .../main/resources/container-log4j.properties | 29 +- .../nodemanager/DummyContainerManager.java | 5 +- .../nodemanager/MockContainerLocalizer.java | 41 + .../TestDefaultContainerExecutor.java | 149 +- .../nodemanager/TestDirectoryCollection.java | 75 +- .../TestDockerContainerExecutor.java | 213 ++ .../TestDockerContainerExecutorWithMocks.java | 259 ++ .../yarn/server/nodemanager/TestEventFlow.java | 2 +- .../nodemanager/TestLinuxContainerExecutor.java | 72 +- .../TestLinuxContainerExecutorWithMocks.java | 2 +- .../TestLocalDirsHandlerService.java | 55 +- .../nodemanager/TestNodeHealthService.java | 2 +- .../nodemanager/TestNodeManagerReboot.java | 68 +- .../nodemanager/TestNodeManagerResync.java | 2 +- .../nodemanager/TestNodeManagerShutdown.java | 2 +- .../nodemanager/TestNodeStatusUpdater.java | 139 +- .../containermanager/TestAuxServices.java | 2 +- .../containermanager/TestContainerManager.java | 14 +- .../TestContainerManagerRecovery.java | 12 +- .../container/TestContainer.java | 29 +- .../launcher/TestContainerLaunch.java | 127 +- .../TestLocalCacheDirectoryManager.java | 11 +- .../localizer/TestLocalizedResource.java | 14 +- .../TestResourceLocalizationService.java | 464 ++- .../TestSharedCacheUploadService.java | 50 + .../sharedcache/TestSharedCacheUploader.java | 241 ++ .../TestLogAggregationService.java | 321 +- .../TestNonAggregatingLogHandler.java | 382 +- .../monitor/TestContainersMonitor.java | 2 +- .../metrics/TestNodeManagerMetrics.java | 18 +- .../TestNMLeveldbStateStoreService.java | 9 +- .../util/TestCgroupsLCEResourcesHandler.java | 7 +- .../webapp/TestContainerLogsPage.java | 82 + .../nodemanager/webapp/TestNMWebServices.java | 4 +- .../webapp/TestNMWebServicesApps.java | 4 +- .../webapp/TestNMWebServicesContainers.java | 4 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 5 + .../server/resourcemanager/AdminService.java | 170 +- .../ApplicationMasterService.java | 112 +- .../server/resourcemanager/ClientRMService.java | 55 +- .../server/resourcemanager/ClusterMetrics.java | 13 +- .../resourcemanager/NodesListManager.java | 10 + .../resourcemanager/RMActiveServiceContext.java | 455 +++ .../server/resourcemanager/RMAppManager.java | 103 +- .../server/resourcemanager/RMAuditLogger.java | 1 + .../yarn/server/resourcemanager/RMContext.java | 11 + .../server/resourcemanager/RMContextImpl.java | 250 +- .../resourcemanager/RMFatalEventType.java | 1 - .../server/resourcemanager/RMServerUtils.java | 46 +- .../server/resourcemanager/ResourceManager.java | 177 +- .../resourcemanager/ResourceTrackerService.java | 41 +- .../metrics/SystemMetricsPublisher.java | 27 - .../ProportionalCapacityPreemptionPolicy.java | 170 +- .../nodelabels/MemoryRMNodeLabelsManager.java | 82 + .../nodelabels/RMNodeLabelsManager.java | 497 +++ .../recovery/FileSystemRMStateStore.java | 202 +- .../recovery/LeveldbRMStateStore.java | 692 ++++ .../recovery/MemoryRMStateStore.java | 123 +- .../recovery/NullRMStateStore.java | 14 +- .../resourcemanager/recovery/RMStateStore.java | 555 +-- .../recovery/RMStateStoreAMRMTokenEvent.java | 46 + .../recovery/RMStateStoreAppAttemptEvent.java | 8 +- .../recovery/RMStateStoreAppEvent.java | 8 +- .../recovery/RMStateStoreEventType.java | 11 +- .../recovery/RMStateStoreRMDTEvent.java | 45 + .../RMStateStoreRMDTMasterKeyEvent.java | 39 + .../recovery/RMStateStoreRemoveAppEvent.java | 8 +- .../recovery/RMStateUpdateAppAttemptEvent.java | 9 +- .../recovery/RMStateUpdateAppEvent.java | 8 +- .../recovery/ZKRMStateStore.java | 141 +- .../records/ApplicationAttemptStateData.java | 36 +- .../recovery/records/ApplicationStateData.java | 25 +- .../RMDelegationTokenIdentifierData.java | 61 + .../pb/ApplicationAttemptStateDataPBImpl.java | 60 +- .../reservation/AbstractReservationSystem.java | 109 +- .../AbstractSchedulerPlanFollower.java | 411 ++ .../reservation/CapacityOverTimePolicy.java | 14 +- .../reservation/CapacityReservationSystem.java | 113 +- .../CapacitySchedulerPlanFollower.java | 358 +- .../reservation/FairReservationSystem.java | 90 + .../reservation/FairSchedulerPlanFollower.java | 141 + .../reservation/NoOverCommitPolicy.java | 4 +- .../resourcemanager/reservation/Planner.java | 3 +- .../reservation/ReservationConstants.java | 28 + .../ReservationSchedulerConfiguration.java | 159 + .../reservation/SharingPolicy.java | 3 +- .../reservation/SimpleCapacityReplanner.java | 14 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 85 +- .../rmapp/RMAppRecoverEvent.java | 36 + .../rmapp/attempt/RMAppAttemptImpl.java | 93 +- .../rmcontainer/RMContainerImpl.java | 9 +- .../server/resourcemanager/rmnode/RMNode.java | 9 +- .../resourcemanager/rmnode/RMNodeImpl.java | 20 +- .../scheduler/AbstractYarnScheduler.java | 108 +- .../server/resourcemanager/scheduler/Queue.java | 19 + .../scheduler/QueueNotFoundException.java | 32 + .../scheduler/SchedulerApplicationAttempt.java | 2 +- .../scheduler/SchedulerNode.java | 29 +- .../scheduler/SchedulerUtils.java | 122 +- .../scheduler/YarnScheduler.java | 5 + .../scheduler/capacity/AbstractCSQueue.java | 457 +++ .../scheduler/capacity/CSQueue.java | 69 +- .../scheduler/capacity/CSQueueUtils.java | 57 +- .../scheduler/capacity/CapacityScheduler.java | 157 +- .../CapacitySchedulerConfiguration.java | 189 +- .../scheduler/capacity/LeafQueue.java | 633 ++-- .../scheduler/capacity/ParentQueue.java | 399 +- .../scheduler/capacity/PlanQueue.java | 10 +- .../scheduler/capacity/ReservationQueue.java | 2 +- .../common/fica/FiCaSchedulerNode.java | 11 +- .../event/NodeLabelsUpdateSchedulerEvent.java | 37 + .../scheduler/event/SchedulerEventType.java | 1 + .../scheduler/fair/AllocationConfiguration.java | 81 +- .../fair/AllocationFileLoaderService.java | 61 +- .../scheduler/fair/FSAppAttempt.java | 2 +- .../scheduler/fair/FSLeafQueue.java | 302 +- .../resourcemanager/scheduler/fair/FSQueue.java | 35 +- .../scheduler/fair/FairScheduler.java | 176 +- .../fair/FairSchedulerConfiguration.java | 3 +- .../scheduler/fair/MaxRunningAppsEnforcer.java | 8 +- .../scheduler/fair/QueueManager.java | 15 +- .../scheduler/fair/QueuePlacementRule.java | 30 +- .../fair/ReservationQueueConfiguration.java | 116 + .../scheduler/fair/SchedulingPolicy.java | 8 +- .../scheduler/fifo/FifoScheduler.java | 34 +- .../security/AMRMTokenSecretManager.java | 12 +- .../security/DelegationTokenRenewer.java | 316 +- .../security/QueueACLsManager.java | 7 + .../security/RMAuthenticationHandler.java | 157 - .../RMDelegationTokenSecretManager.java | 15 +- .../server/resourcemanager/webapp/AppBlock.java | 50 +- .../resourcemanager/webapp/AppsBlock.java | 7 +- .../webapp/CapacitySchedulerPage.java | 3 +- .../webapp/DefaultSchedulerPage.java | 4 +- .../webapp/FairSchedulerAppsBlock.java | 9 +- .../webapp/JAXBContextResolver.java | 2 +- .../webapp/MetricsOverviewTable.java | 9 +- .../server/resourcemanager/webapp/NavBlock.java | 3 +- .../resourcemanager/webapp/NodeLabelsPage.java | 91 + .../resourcemanager/webapp/NodesPage.java | 148 +- .../server/resourcemanager/webapp/RMWebApp.java | 8 +- .../resourcemanager/webapp/RMWebServices.java | 319 +- .../resourcemanager/webapp/RmController.java | 6 +- .../resourcemanager/webapp/dao/AppQueue.java | 46 + .../dao/ApplicationSubmissionContextInfo.java | 23 + .../webapp/dao/CapacitySchedulerQueueInfo.java | 16 + .../webapp/dao/ClusterMetricsInfo.java | 3 +- .../webapp/dao/FairSchedulerLeafQueueInfo.java | 14 +- .../webapp/dao/FairSchedulerQueueInfo.java | 9 +- .../resourcemanager/webapp/dao/NodeInfo.java | 15 + .../webapp/dao/NodeLabelsInfo.java | 52 + .../webapp/dao/NodeToLabelsInfo.java | 41 + .../webapp/dao/UserMetricsInfo.java | 3 +- .../yarn_server_resourcemanager_recovery.proto | 6 + .../server/resourcemanager/Application.java | 1 + .../yarn/server/resourcemanager/MockAM.java | 95 +- .../yarn/server/resourcemanager/MockNM.java | 2 +- .../yarn/server/resourcemanager/MockNodes.java | 52 +- .../yarn/server/resourcemanager/MockRM.java | 65 +- .../RMDelegationTokenIdentifierForTest.java | 156 +- .../server/resourcemanager/RMHATestBase.java | 5 +- .../server/resourcemanager/TestAppManager.java | 16 +- .../resourcemanager/TestApplicationACLs.java | 3 +- .../resourcemanager/TestApplicationCleanup.java | 2 +- .../TestApplicationMasterLauncher.java | 41 +- .../TestApplicationMasterService.java | 32 +- .../resourcemanager/TestClientRMService.java | 182 +- .../resourcemanager/TestClusterMetrics.java | 60 + .../TestContainerResourceUsage.java | 39 +- .../resourcemanager/TestFifoScheduler.java | 2 + .../yarn/server/resourcemanager/TestRM.java | 120 +- .../resourcemanager/TestRMAdminService.java | 19 + .../yarn/server/resourcemanager/TestRMHA.java | 189 +- .../resourcemanager/TestRMProxyUsersConf.java | 106 + .../server/resourcemanager/TestRMRestart.java | 221 +- .../resourcemanager/TestResourceManager.java | 8 +- .../TestResourceTrackerService.java | 22 +- .../TestWorkPreservingRMRestart.java | 321 +- .../ahs/TestRMApplicationHistoryWriter.java | 12 +- .../applicationsmanager/MockAsm.java | 2 +- .../TestAMRMRPCResponseId.java | 14 +- .../applicationsmanager/TestAMRestart.java | 30 +- .../metrics/TestSystemMetricsPublisher.java | 9 +- ...estProportionalCapacityPreemptionPolicy.java | 285 +- .../nodelabels/TestRMNodeLabelsManager.java | 464 +++ .../recovery/RMStateStoreTestBase.java | 126 +- .../recovery/TestFSRMStateStore.java | 54 + .../recovery/TestLeveldbRMStateStore.java | 139 + .../recovery/TestZKRMStateStore.java | 179 + .../reservation/ReservationSystemTestUtil.java | 189 +- .../reservation/TestCapacityOverTimePolicy.java | 21 +- .../TestCapacityReservationSystem.java | 33 +- .../TestCapacitySchedulerPlanFollower.java | 210 +- .../reservation/TestFairReservationSystem.java | 128 + .../TestFairSchedulerPlanFollower.java | 203 + .../reservation/TestGreedyReservationAgent.java | 30 +- .../reservation/TestNoOverCommitPolicy.java | 15 +- .../TestSchedulerPlanFollowerBase.java | 191 + .../TestSimpleCapacityReplanner.java | 8 +- .../rmapp/TestRMAppTransitions.java | 104 +- .../attempt/TestRMAppAttemptTransitions.java | 72 +- .../rmcontainer/TestRMContainerImpl.java | 8 +- .../scheduler/TestAbstractYarnScheduler.java | 292 ++ .../TestSchedulerApplicationAttempt.java | 2 +- .../scheduler/TestSchedulerUtils.java | 283 +- .../capacity/TestApplicationLimits.java | 11 +- .../scheduler/capacity/TestCSQueueUtils.java | 28 +- .../capacity/TestCapacityScheduler.java | 32 +- .../TestCapacitySchedulerDynamicBehavior.java | 3 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 193 + .../scheduler/capacity/TestChildQueueOrder.java | 5 +- .../capacity/TestContainerAllocation.java | 479 ++- .../scheduler/capacity/TestLeafQueue.java | 243 +- .../scheduler/capacity/TestParentQueue.java | 5 +- .../scheduler/capacity/TestQueueMappings.java | 10 +- .../scheduler/capacity/TestQueueParsing.java | 587 ++- .../capacity/TestReservationQueue.java | 9 +- .../scheduler/capacity/TestReservations.java | 30 +- .../scheduler/capacity/TestUtils.java | 34 +- .../scheduler/fair/FairSchedulerTestBase.java | 6 +- .../scheduler/fair/PeriodGroupsMapping.java | 44 + .../fair/TestAllocationFileLoaderService.java | 105 + .../scheduler/fair/TestFSLeafQueue.java | 93 +- .../scheduler/fair/TestFairScheduler.java | 280 +- .../fair/TestFairSchedulerConfiguration.java | 6 + .../fair/TestFairSchedulerPreemption.java | 3 +- .../fair/TestMaxRunningAppsEnforcer.java | 64 +- .../fair/TestQueuePlacementPolicy.java | 49 +- .../security/TestAMRMTokens.java | 2 + .../security/TestClientToAMTokens.java | 122 +- .../security/TestDelegationTokenRenewer.java | 275 +- .../resourcemanager/webapp/TestAppPage.java | 90 + .../resourcemanager/webapp/TestNodesPage.java | 47 +- .../resourcemanager/webapp/TestRMWebApp.java | 42 +- .../webapp/TestRMWebServices.java | 12 +- .../webapp/TestRMWebServicesApps.java | 11 +- .../TestRMWebServicesAppsModification.java | 187 +- .../webapp/TestRMWebServicesCapacitySched.java | 15 +- ...ebServicesDelegationTokenAuthentication.java | 151 +- .../TestRMWebServicesDelegationTokens.java | 16 +- .../webapp/TestRMWebServicesFairScheduler.java | 11 +- ...tRMWebServicesHttpStaticUserPermissions.java | 195 + .../webapp/TestRMWebServicesNodeLabels.java | 354 ++ .../webapp/TestRMWebServicesNodes.java | 11 +- .../pom.xml | 4 + .../sharedcachemanager/CleanerService.java | 218 ++ .../server/sharedcachemanager/CleanerTask.java | 308 ++ .../ClientProtocolService.java | 192 + .../SCMAdminProtocolService.java | 146 + .../sharedcachemanager/SharedCacheManager.java | 77 + .../SharedCacheUploaderService.java | 140 + .../metrics/CleanerMetrics.java | 150 + .../metrics/ClientSCMMetrics.java | 90 + .../metrics/SharedCacheUploaderMetrics.java | 82 + .../store/InMemorySCMStore.java | 520 +++ .../sharedcachemanager/store/SCMStore.java | 213 ++ .../store/SharedCacheResource.java | 64 + .../store/SharedCacheResourceReference.java | 86 + .../webapp/SCMController.java | 43 + .../webapp/SCMMetricsInfo.java | 70 + .../webapp/SCMOverviewPage.java | 95 + .../sharedcachemanager/webapp/SCMWebServer.java | 91 + .../sharedcachemanager/DummyAppChecker.java | 43 + .../sharedcachemanager/TestCleanerTask.java | 152 + .../TestClientSCMProtocolService.java | 278 ++ .../TestSCMAdminProtocolService.java | 135 + .../TestSharedCacheUploaderService.java | 188 + .../metrics/TestCleanerMetrics.java | 64 + .../store/SCMStoreBaseTest.java | 41 + .../store/TestInMemorySCMStore.java | 317 ++ .../hadoop/yarn/server/MiniYARNCluster.java | 6 +- .../server/TestContainerManagerSecurity.java | 12 +- .../server/webproxy/WebAppProxyServlet.java | 15 +- .../src/site/apt/DockerContainerExecutor.apt.vm | 204 + .../src/site/apt/FairScheduler.apt.vm | 17 +- .../src/site/apt/NodeManagerCgroups.apt.vm | 77 + .../src/site/apt/NodeManagerRestart.apt.vm | 86 + .../src/site/apt/ResourceManagerRest.apt.vm | 161 +- .../src/site/apt/SecureContainer.apt.vm | 68 +- .../src/site/apt/TimelineServer.apt.vm | 4 +- .../src/site/apt/YarnCommands.apt.vm | 269 +- .../hadoop-yarn-site/src/site/apt/index.apt.vm | 48 +- .../src/site/markdown/registry/index.md | 28 + .../site/markdown/registry/registry-security.md | 120 + .../registry/using-the-yarn-service-registry.md | 150 + .../src/site/markdown/registry/yarn-registry.md | 1020 +++++ hadoop-yarn-project/hadoop-yarn/pom.xml | 1 + pom.xml | 12 +- 1640 files changed, 117716 insertions(+), 28212 deletions(-) ----------------------------------------------------------------------
