Merge commit '456e901a4c5c639267ee87b8e5f1319f256d20c2' (HDFS-6407. Add sorting and pagination in the datanode tab of the NN Web UI. Contributed by Haohui Mai.) into HDFS-7285-merge
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b6a63bb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b6a63bb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b6a63bb Branch: refs/heads/HDFS-7285 Commit: 6b6a63bbbda920315d3d24b61ed3344a78a981b6 Parents: b57c9a3 456e901 Author: Zhe Zhang <zhezh...@cloudera.com> Authored: Wed Aug 19 22:52:32 2015 -0700 Committer: Zhe Zhang <zhezh...@cloudera.com> Committed: Mon Aug 24 12:59:26 2015 -0700 ---------------------------------------------------------------------- .gitignore | 1 + BUILDING.txt | 39 +- dev-support/determine-flaky-tests-hadoop.py | 57 +- dev-support/docker/Dockerfile | 8 + dev-support/releasedocmaker.py | 405 +- dev-support/smart-apply-patch.sh | 52 +- dev-support/test-patch.d/shellcheck.sh | 2 +- dev-support/test-patch.sh | 310 +- hadoop-client/pom.xml | 8 + .../server/AuthenticationFilter.java | 28 +- .../src/site/markdown/Configuration.md | 18 +- hadoop-common-project/hadoop-common/CHANGES.txt | 368 +- .../hadoop-common/HadoopCommon.cmake | 207 + .../hadoop-common/HadoopJNI.cmake | 97 + hadoop-common-project/hadoop-common/pom.xml | 41 +- .../hadoop-common/src/CMakeLists.txt | 366 +- .../hadoop-common/src/JNIFlags.cmake | 124 - .../hadoop-common/src/main/bin/hadoop | 62 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 73 +- .../src/main/bin/hadoop-functions.sh | 423 +- .../hadoop-common/src/main/bin/slaves.sh | 3 +- .../main/conf/hadoop-user-functions.sh.example | 29 +- .../src/main/conf/log4j.properties | 21 +- .../src/main/docs/changes/ChangesFancyStyle.css | 170 - .../main/docs/changes/ChangesSimpleStyle.css | 49 - .../src/main/docs/changes/changes2html.pl | 286 - .../src/main/docs/releasenotes.html | 29099 ----------------- .../src/main/docs/src/documentation/README.txt | 7 - .../classes/CatalogManager.properties | 40 - .../main/docs/src/documentation/conf/cli.xconf | 327 - .../src/documentation/content/xdocs/index.xml | 48 - .../src/documentation/content/xdocs/site.xml | 263 - .../src/documentation/content/xdocs/tabs.xml | 37 - .../resources/images/architecture.gif | Bin 15461 -> 0 bytes .../resources/images/common-logo.jpg | Bin 5887 -> 0 bytes .../resources/images/core-logo.gif | Bin 6665 -> 0 bytes .../documentation/resources/images/favicon.ico | Bin 766 -> 0 bytes .../resources/images/hadoop-logo-big.jpg | Bin 127869 -> 0 bytes .../resources/images/hadoop-logo.jpg | Bin 9443 -> 0 bytes .../resources/images/hdfsarchitecture.gif | Bin 17653 -> 0 bytes .../resources/images/hdfsarchitecture.odg | Bin 41298 -> 0 bytes .../resources/images/hdfsarchitecture.png | Bin 40571 -> 0 bytes .../resources/images/hdfsdatanodes.gif | Bin 16060 -> 0 bytes .../resources/images/hdfsdatanodes.odg | Bin 37296 -> 0 bytes .../resources/images/hdfsdatanodes.png | Bin 30012 -> 0 bytes .../main/docs/src/documentation/skinconf.xml | 366 - .../hadoop-common/src/main/docs/status.xml | 75 - .../org/apache/hadoop/conf/Configuration.java | 8 +- .../apache/hadoop/conf/ReconfigurableBase.java | 14 +- .../crypto/key/kms/KMSClientProvider.java | 4 +- .../apache/hadoop/fs/AbstractFileSystem.java | 13 + .../org/apache/hadoop/fs/BlockLocation.java | 55 +- .../apache/hadoop/fs/ByteBufferReadable.java | 4 + .../fs/CommonConfigurationKeysPublic.java | 6 + .../apache/hadoop/fs/DelegateToFileSystem.java | 19 +- .../java/org/apache/hadoop/fs/FileContext.java | 23 + .../java/org/apache/hadoop/fs/FileStatus.java | 15 +- .../java/org/apache/hadoop/fs/FileSystem.java | 196 +- .../java/org/apache/hadoop/fs/FileUtil.java | 206 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 6 + .../java/org/apache/hadoop/fs/FilterFs.java | 6 + .../main/java/org/apache/hadoop/fs/FsShell.java | 33 +- .../main/java/org/apache/hadoop/fs/Globber.java | 28 +- .../org/apache/hadoop/fs/LocatedFileStatus.java | 10 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 156 +- .../java/org/apache/hadoop/fs/StorageType.java | 7 +- .../apache/hadoop/fs/TrashPolicyDefault.java | 7 +- .../hadoop/fs/permission/FsPermission.java | 7 +- .../hadoop/fs/permission/UmaskParser.java | 2 +- .../hadoop/fs/sftp/SFTPConnectionPool.java | 303 + .../apache/hadoop/fs/sftp/SFTPFileSystem.java | 671 + .../apache/hadoop/fs/sftp/SFTPInputStream.java | 130 + .../org/apache/hadoop/fs/sftp/package-info.java | 19 + .../java/org/apache/hadoop/fs/shell/Delete.java | 17 +- .../org/apache/hadoop/fs/shell/FsCommand.java | 30 +- .../java/org/apache/hadoop/fs/shell/Ls.java | 8 + .../java/org/apache/hadoop/fs/shell/Mkdir.java | 3 +- .../java/org/apache/hadoop/fs/shell/Touch.java | 84 + .../java/org/apache/hadoop/fs/shell/Touchz.java | 84 - .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 + .../org/apache/hadoop/fs/viewfs/ViewFs.java | 15 + .../fs/viewfs/ViewFsLocatedFileStatus.java | 3 +- .../apache/hadoop/ha/ActiveStandbyElector.java | 20 +- .../apache/hadoop/ha/ZKFailoverController.java | 63 +- .../main/java/org/apache/hadoop/io/MD5Hash.java | 3 +- .../org/apache/hadoop/io/ReadaheadPool.java | 4 +- .../java/org/apache/hadoop/io/SequenceFile.java | 85 +- .../main/java/org/apache/hadoop/io/Text.java | 4 +- .../hadoop/io/compress/bzip2/Bzip2Factory.java | 2 +- .../org/apache/hadoop/io/nativeio/NativeIO.java | 98 +- .../apache/hadoop/io/retry/MultiException.java | 49 + .../hadoop/io/retry/RetryInvocationHandler.java | 99 +- .../apache/hadoop/io/retry/RetryPolicies.java | 6 +- .../org/apache/hadoop/io/retry/RetryUtils.java | 7 +- .../org/apache/hadoop/ipc/CallQueueManager.java | 27 +- .../main/java/org/apache/hadoop/ipc/Client.java | 8 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 2 +- .../org/apache/hadoop/ipc/RpcClientUtil.java | 24 + .../main/java/org/apache/hadoop/ipc/Server.java | 5 +- .../apache/hadoop/ipc/metrics/RpcMetrics.java | 6 +- .../org/apache/hadoop/jmx/package-info.java | 5 +- .../metrics2/impl/MetricsSinkAdapter.java | 6 +- .../hadoop/metrics2/impl/MetricsSystemImpl.java | 2 +- .../org/apache/hadoop/net/NetworkTopology.java | 96 +- .../hadoop/net/unix/DomainSocketWatcher.java | 21 +- .../hadoop/security/LdapGroupsMapping.java | 29 +- .../apache/hadoop/security/ProviderUtils.java | 30 + .../apache/hadoop/security/SaslPlainServer.java | 2 +- .../hadoop/security/UserGroupInformation.java | 3 +- .../alias/AbstractJavaKeyStoreProvider.java | 339 + .../security/alias/JavaKeyStoreProvider.java | 257 +- .../alias/LocalJavaKeyStoreProvider.java | 215 + .../hadoop/security/token/SecretManager.java | 2 +- .../org/apache/hadoop/security/token/Token.java | 2 +- .../hadoop/security/token/TokenIdentifier.java | 2 +- .../apache/hadoop/security/token/TokenInfo.java | 2 +- .../hadoop/security/token/TokenRenewer.java | 2 +- .../hadoop/security/token/TokenSelector.java | 2 +- .../DelegationTokenAuthenticationFilter.java | 16 +- .../hadoop/security/token/package-info.java | 2 +- .../apache/hadoop/tracing/SpanReceiverHost.java | 45 +- .../java/org/apache/hadoop/util/ConfTest.java | 368 + .../org/apache/hadoop/util/CpuTimeTracker.java | 115 + .../org/apache/hadoop/util/DiskChecker.java | 24 +- .../org/apache/hadoop/util/JvmPauseMonitor.java | 12 +- .../java/org/apache/hadoop/util/LineReader.java | 8 + .../org/apache/hadoop/util/ReflectionUtils.java | 6 +- .../java/org/apache/hadoop/util/RunJar.java | 3 +- .../main/java/org/apache/hadoop/util/Shell.java | 22 +- .../org/apache/hadoop/util/StringUtils.java | 8 + .../java/org/apache/hadoop/util/SysInfo.java | 137 + .../org/apache/hadoop/util/SysInfoLinux.java | 690 + .../org/apache/hadoop/util/SysInfoWindows.java | 208 + .../apache/hadoop/util/hash/JenkinsHash.java | 15 +- .../org/apache/hadoop/io/nativeio/NativeIO.c | 150 +- .../org/apache/hadoop/net/unix/DomainSocket.c | 17 +- .../hadoop/net/unix/DomainSocketWatcher.c | 2 +- .../src/org/apache/hadoop/util/NativeCrc32.c | 4 +- ...oop.security.alias.CredentialProviderFactory | 1 + .../src/main/resources/core-default.xml | 44 + .../src/site/markdown/CLIMiniCluster.md.vm | 2 +- .../src/site/markdown/CommandsManual.md | 4 +- .../src/site/markdown/DeprecatedProperties.md | 1 + .../src/site/markdown/FileSystemShell.md | 4 +- .../hadoop-common/src/site/markdown/Metrics.md | 7 + .../hadoop-common/src/site/markdown/Tracing.md | 12 +- .../src/site/markdown/filesystem/filesystem.md | 4 + .../apache/hadoop/conf/TestConfiguration.java | 15 + .../apache/hadoop/conf/TestReconfiguration.java | 12 +- .../apache/hadoop/crypto/TestCryptoCodec.java | 11 +- ...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 11 +- .../apache/hadoop/fs/FCStatisticsBaseTest.java | 56 +- .../fs/FileContextMainOperationsBaseTest.java | 4 +- .../hadoop/fs/FileSystemContractBaseTest.java | 11 +- .../org/apache/hadoop/fs/SymlinkBaseTest.java | 45 +- .../org/apache/hadoop/fs/TestBlockLocation.java | 23 +- .../org/apache/hadoop/fs/TestFileStatus.java | 22 + .../java/org/apache/hadoop/fs/TestFileUtil.java | 112 +- .../java/org/apache/hadoop/fs/TestFsShell.java | 28 + .../apache/hadoop/fs/TestFsShellReturnCode.java | 60 +- .../org/apache/hadoop/fs/TestHarFileSystem.java | 3 + .../apache/hadoop/fs/TestLocalFileSystem.java | 83 +- .../fs/TestLocalFileSystemPermission.java | 111 +- .../apache/hadoop/fs/TestSymlinkLocalFS.java | 18 + .../java/org/apache/hadoop/fs/TestTrash.java | 14 +- .../fs/contract/AbstractContractAppendTest.java | 4 +- .../AbstractContractGetFileStatusTest.java | 61 + .../contract/AbstractContractSetTimesTest.java | 61 + .../hadoop/fs/contract/ContractOptions.java | 12 + .../TestLocalFSContractGetFileStatus.java | 33 + .../localfs/TestLocalFSContractSetTimes.java | 33 + .../TestRawlocalContractGetFileStatus.java | 33 + .../rawlocal/TestRawlocalContractSetTimes.java | 33 + .../hadoop/fs/sftp/TestSFTPFileSystem.java | 308 + .../hadoop/fs/shell/TestCopyPreserveFlag.java | 63 +- .../org/apache/hadoop/fs/shell/TestCount.java | 4 +- .../java/org/apache/hadoop/fs/shell/TestLs.java | 25 + .../org/apache/hadoop/fs/shell/TestMove.java | 1 + .../org/apache/hadoop/ha/MiniZKFCCluster.java | 93 +- .../hadoop/ha/TestActiveStandbyElector.java | 31 + .../hadoop/ha/TestZKFailoverController.java | 32 + .../http/TestAuthenticationSessionCookie.java | 187 + .../apache/hadoop/http/TestHttpCookieFlag.java | 2 +- .../hadoop/io/TestSequenceFileAppend.java | 316 + .../apache/hadoop/io/nativeio/TestNativeIO.java | 90 +- .../hadoop/io/retry/TestDefaultRetryPolicy.java | 101 + .../apache/hadoop/ipc/TestCallQueueManager.java | 6 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 78 +- .../metrics2/impl/TestMetricsSystemImpl.java | 60 + .../org/apache/hadoop/net/ServerSocketUtil.java | 63 + .../apache/hadoop/net/TestClusterTopology.java | 75 +- .../net/unix/TestDomainSocketWatcher.java | 75 + .../TestLdapGroupsMappingWithPosixGroup.java | 25 +- .../alias/TestCredentialProviderFactory.java | 42 +- .../apache/hadoop/test/GenericTestUtils.java | 12 + .../apache/hadoop/tracing/SetSpanReceiver.java | 109 + .../org/apache/hadoop/util/TestConfTest.java | 204 + .../org/apache/hadoop/util/TestDiskChecker.java | 22 + .../org/apache/hadoop/util/TestStringUtils.java | 7 +- .../apache/hadoop/util/TestSysInfoLinux.java | 432 + .../apache/hadoop/util/TestSysInfoWindows.java | 100 + .../src/test/resources/contract/localfs.xml | 10 + .../src/test/resources/contract/rawlocal.xml | 10 + .../scripts/hadoop-functions_test_helper.bash | 56 + .../src/test/scripts/hadoop_add_classpath.bats | 100 + .../src/test/scripts/hadoop_add_colonpath.bats | 96 + .../scripts/hadoop_add_common_to_classpath.bats | 71 + .../test/scripts/hadoop_add_javalibpath.bats | 98 + .../src/test/scripts/hadoop_add_ldlibpath.bats | 97 + .../src/test/scripts/hadoop_add_param.bats | 49 + .../hadoop_add_to_classpath_userpath.bats | 98 + .../src/test/scripts/hadoop_basic_init.bats | 94 + .../src/test/scripts/hadoop_bootstrap.bats | 51 + .../src/test/scripts/hadoop_confdir.bats | 92 + .../test/scripts/hadoop_deprecate_envvar.bats | 32 + .../src/test/scripts/hadoop_finalize.bats | 206 + .../scripts/hadoop_finalize_catalina_opts.bats | 56 + .../test/scripts/hadoop_finalize_classpath.bats | 64 + .../scripts/hadoop_finalize_hadoop_heap.bats | 87 + .../scripts/hadoop_finalize_hadoop_opts.bats | 52 + .../test/scripts/hadoop_finalize_libpaths.bats | 30 + .../src/test/scripts/hadoop_java_setup.bats | 47 + .../src/test/scripts/hadoop_os_tricks.bats | 34 + .../src/test/scripts/hadoop_rotate_log.bats | 52 + .../src/test/scripts/hadoop_shellprofile.bats | 91 + .../src/test/scripts/hadoop_slaves.bats | 37 + .../src/test/scripts/hadoop_ssh.bats | 51 + .../scripts/hadoop_translate_cygwin_path.bats | 48 + .../test/scripts/hadoop_validate_classname.bats | 26 + .../hadoop-common/src/test/scripts/run-bats.sh | 43 + hadoop-common-project/hadoop-kms/pom.xml | 4 +- .../crypto/key/kms/server/KMSMDCFilter.java | 2 +- .../hadoop-kms/src/main/sbin/kms.sh | 29 +- hadoop-dist/pom.xml | 4 +- .../dev-support/findbugsExcludeFile.xml | 4 + hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 53 + .../org/apache/hadoop/hdfs/DFSUtilClient.java | 9 +- .../hdfs/client/HdfsClientConfigKeys.java | 2 + .../org/apache/hadoop/hdfs/inotify/Event.java | 132 +- .../hdfs/protocol/BlockStoragePolicy.java | 12 +- .../hadoop/hdfs/protocol/CachePoolInfo.java | 3 - .../hadoop/hdfs/protocol/ClientProtocol.java | 1511 + .../hadoop/hdfs/protocol/ErasureCodingZone.java | 54 + .../hadoop/hdfs/protocol/HdfsConstants.java | 3 + .../hdfs/server/namenode/SafeModeException.java | 38 + .../hadoop/hdfs/web/ByteRangeInputStream.java | 57 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 24 + .../org/apache/hadoop/hdfs/web/TokenAspect.java | 16 +- .../hadoop/hdfs/web/URLConnectionFactory.java | 13 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 62 +- .../src/main/proto/ClientDatanodeProtocol.proto | 247 + .../src/main/proto/ClientNamenodeProtocol.proto | 880 + .../hadoop-hdfs-client/src/main/proto/acl.proto | 108 + .../src/main/proto/datatransfer.proto | 304 + .../src/main/proto/encryption.proto | 67 + .../src/main/proto/erasurecoding.proto | 68 + .../src/main/proto/hdfs.proto | 648 + .../src/main/proto/inotify.proto | 133 + .../src/main/proto/xattr.proto | 75 + hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 8 +- .../http/server/HttpFSAuthenticationFilter.java | 5 + .../apache/hadoop/lib/servlet/ServerWebApp.java | 3 +- .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 29 +- .../org/apache/hadoop/test/TestDirHelper.java | 2 +- .../org/apache/hadoop/test/TestHdfsHelper.java | 4 +- .../org/apache/hadoop/test/TestJettyHelper.java | 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 477 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 19 +- .../hadoop-hdfs/src/CMakeLists.txt | 129 +- .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 2 +- .../bkjournal/TestBookKeeperHACheckpoints.java | 7 +- .../src/contrib/libwebhdfs/CMakeLists.txt | 8 +- .../libwebhdfs/resources/FindJansson.cmake | 9 +- .../hadoop-hdfs/src/main/bin/hdfs | 97 +- .../hadoop-hdfs/src/main/bin/hdfs.cmd | 13 +- .../hadoop-hdfs/src/main/bin/start-balancer.sh | 12 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 10 +- .../apache/hadoop/fs/BlockStorageLocation.java | 1 + .../main/java/org/apache/hadoop/fs/Hdfs.java | 5 + .../java/org/apache/hadoop/fs/SWebHdfs.java | 51 + .../main/java/org/apache/hadoop/fs/WebHdfs.java | 51 + .../main/java/org/apache/hadoop/fs/package.html | 26 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 57 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 40 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 18 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 196 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 2 +- .../hadoop/hdfs/DistributedFileSystem.java | 25 + .../java/org/apache/hadoop/hdfs/HAUtil.java | 36 +- .../org/apache/hadoop/hdfs/NameNodeProxies.java | 43 +- .../hadoop/hdfs/client/impl/LeaseRenewer.java | 12 +- .../hdfs/protocol/ClientDatanodeProtocol.java | 5 + .../hadoop/hdfs/protocol/ClientProtocol.java | 1484 - .../hadoop/hdfs/protocol/LayoutVersion.java | 51 +- ...tDatanodeProtocolServerSideTranslatorPB.java | 19 + .../ClientDatanodeProtocolTranslatorPB.java | 19 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 17 + .../ClientNamenodeProtocolTranslatorPB.java | 15 +- .../DatanodeProtocolClientSideTranslatorPB.java | 8 +- .../DatanodeProtocolServerSideTranslatorPB.java | 3 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 20 +- .../token/block/BlockTokenSecretManager.java | 40 +- .../hadoop/hdfs/server/balancer/Balancer.java | 131 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 155 +- .../hadoop/hdfs/server/balancer/ExitStatus.java | 3 +- .../hdfs/server/balancer/MovedBlocks.java | 5 + .../hdfs/server/balancer/NameNodeConnector.java | 16 + .../hdfs/server/blockmanagement/BlockInfo.java | 33 +- .../blockmanagement/BlockInfoContiguous.java | 10 +- .../BlockInfoContiguousUnderConstruction.java | 281 + .../blockmanagement/BlockInfoStriped.java | 8 +- .../BlockInfoStripedUnderConstruction.java | 297 + .../BlockInfoUnderConstructionContiguous.java | 281 - .../BlockInfoUnderConstructionStriped.java | 297 - .../server/blockmanagement/BlockManager.java | 174 +- .../BlockManagerFaultInjector.java | 52 + .../BlockPlacementPolicyDefault.java | 3 +- .../BlockPlacementPolicyRackFaultTolarent.java | 154 - .../BlockPlacementPolicyRackFaultTolerant.java | 154 + .../BlockReportLeaseManager.java | 354 + .../blockmanagement/BlockStatsMXBean.java | 36 + .../BlockStoragePolicySuite.java | 6 +- .../hdfs/server/blockmanagement/BlocksMap.java | 8 +- .../CacheReplicationMonitor.java | 10 +- .../blockmanagement/CorruptReplicasMap.java | 23 +- .../blockmanagement/DatanodeDescriptor.java | 58 +- .../server/blockmanagement/DatanodeManager.java | 22 +- .../blockmanagement/DatanodeStatistics.java | 6 + .../blockmanagement/DatanodeStorageInfo.java | 7 +- .../blockmanagement/DecommissionManager.java | 42 +- .../blockmanagement/HeartbeatManager.java | 96 +- .../blockmanagement/InvalidateBlocks.java | 7 +- .../blockmanagement/StorageTypeStats.java | 115 + .../blockmanagement/UnderReplicatedBlocks.java | 9 +- .../hdfs/server/common/HdfsServerConstants.java | 3 - .../hadoop/hdfs/server/common/Storage.java | 24 +- .../hdfs/server/datanode/BPServiceActor.java | 76 +- .../hdfs/server/datanode/BlockReceiver.java | 160 +- .../hdfs/server/datanode/BlockScanner.java | 38 +- .../hdfs/server/datanode/BlockSender.java | 11 +- .../hadoop/hdfs/server/datanode/DNConf.java | 17 +- .../hadoop/hdfs/server/datanode/DataNode.java | 81 +- .../hdfs/server/datanode/DataStorage.java | 47 +- .../hdfs/server/datanode/DataXceiver.java | 57 +- .../hdfs/server/datanode/ReplicaInPipeline.java | 15 +- .../datanode/ReplicaInPipelineInterface.java | 10 + .../hdfs/server/datanode/StorageLocation.java | 4 +- .../hdfs/server/datanode/VolumeScanner.java | 20 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 149 +- .../server/datanode/fsdataset/FsVolumeSpi.java | 47 +- .../datanode/fsdataset/impl/BlockPoolSlice.java | 31 +- .../datanode/fsdataset/impl/FsDatasetCache.java | 7 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 103 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 7 +- .../server/datanode/web/DatanodeHttpServer.java | 53 +- .../datanode/web/webhdfs/ExceptionHandler.java | 2 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 113 +- .../hdfs/server/namenode/AclTransformation.java | 30 +- .../hdfs/server/namenode/BackupImage.java | 2 +- .../hdfs/server/namenode/CacheManager.java | 24 +- .../hdfs/server/namenode/CheckpointConf.java | 14 +- .../hdfs/server/namenode/Checkpointer.java | 8 +- .../server/namenode/EncryptionZoneManager.java | 7 + .../hdfs/server/namenode/FSDirAppendOp.java | 268 + .../hdfs/server/namenode/FSDirAttrOp.java | 26 +- .../hdfs/server/namenode/FSDirConcatOp.java | 5 +- .../server/namenode/FSDirStatAndListingOp.java | 106 +- .../hdfs/server/namenode/FSDirTruncateOp.java | 370 + .../hdfs/server/namenode/FSDirWriteFileOp.java | 17 +- .../hdfs/server/namenode/FSDirectory.java | 114 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 54 +- .../hdfs/server/namenode/FSEditLogLoader.java | 22 +- .../hdfs/server/namenode/FSEditLogOp.java | 4 +- .../hadoop/hdfs/server/namenode/FSImage.java | 27 +- .../hdfs/server/namenode/FSImageFormat.java | 8 +- .../server/namenode/FSImageFormatPBINode.java | 23 +- .../server/namenode/FSImageFormatProtobuf.java | 33 +- .../server/namenode/FSImageSerialization.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 924 +- .../hdfs/server/namenode/FSNamesystemLock.java | 11 + .../hadoop/hdfs/server/namenode/INode.java | 2 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 11 +- .../hdfs/server/namenode/ImageServlet.java | 88 +- .../namenode/InotifyFSEditLogOpTranslator.java | 4 + .../hdfs/server/namenode/LeaseManager.java | 17 +- .../hdfs/server/namenode/NNUpgradeUtil.java | 18 +- .../hadoop/hdfs/server/namenode/NameNode.java | 5 +- .../server/namenode/NameNodeHttpServer.java | 7 +- .../server/namenode/NameNodeLayoutVersion.java | 48 +- .../hdfs/server/namenode/NameNodeMXBean.java | 5 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 40 +- .../hdfs/server/namenode/NamenodeFsck.java | 7 +- .../hadoop/hdfs/server/namenode/Namesystem.java | 20 +- .../hadoop/hdfs/server/namenode/SafeMode.java | 1 - .../hdfs/server/namenode/SafeModeException.java | 39 - .../hdfs/server/namenode/SecondaryNameNode.java | 4 +- .../hdfs/server/namenode/TransferFsImage.java | 47 +- .../server/namenode/ha/BootstrapStandby.java | 94 +- .../ha/ConfiguredFailoverProxyProvider.java | 52 +- .../hdfs/server/namenode/ha/EditLogTailer.java | 162 +- .../server/namenode/ha/RemoteNameNodeInfo.java | 133 + .../ha/RequestHedgingProxyProvider.java | 186 + .../server/namenode/ha/StandbyCheckpointer.java | 182 +- .../hdfs/server/namenode/ha/StandbyState.java | 3 +- .../namenode/metrics/FSNamesystemMBean.java | 14 + .../snapshot/FSImageFormatPBSnapshot.java | 6 +- .../hdfs/server/namenode/snapshot/FileDiff.java | 1 - .../server/namenode/snapshot/FileDiffList.java | 4 +- .../top/window/RollingWindowManager.java | 8 +- .../web/resources/NamenodeWebHdfsMethods.java | 12 +- .../server/protocol/BlockReportContext.java | 25 +- .../hdfs/server/protocol/DatanodeProtocol.java | 5 +- .../hdfs/server/protocol/HeartbeatResponse.java | 10 +- .../hdfs/server/protocol/RegisterCommand.java | 2 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 75 +- .../hdfs/tools/DFSZKFailoverController.java | 13 + .../org/apache/hadoop/hdfs/tools/DFSck.java | 10 +- .../apache/hadoop/hdfs/tools/DebugAdmin.java | 10 +- .../hdfs/tools/DelegationTokenFetcher.java | 19 +- .../offlineImageViewer/FSImageHandler.java | 60 +- .../tools/offlineImageViewer/FSImageLoader.java | 164 + .../offlineImageViewer/PBImageXmlWriter.java | 20 +- .../src/main/native/fuse-dfs/CMakeLists.txt | 91 +- .../src/main/native/fuse-dfs/fuse_impls_open.c | 4 - .../src/main/native/libhdfs/expect.h | 18 + .../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 1 + .../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 3 +- .../main/native/libhdfs/test/test_libhdfs_ops.c | 9 + .../main/native/libhdfs/test_libhdfs_threaded.c | 17 +- .../src/main/proto/ClientDatanodeProtocol.proto | 235 - .../src/main/proto/ClientNamenodeProtocol.proto | 870 - .../src/main/proto/DatanodeProtocol.proto | 6 + .../hadoop-hdfs/src/main/proto/acl.proto | 113 - .../src/main/proto/datatransfer.proto | 304 - .../hadoop-hdfs/src/main/proto/editlog.proto | 35 + .../hadoop-hdfs/src/main/proto/encryption.proto | 67 - .../src/main/proto/erasurecoding.proto | 68 - .../hadoop-hdfs/src/main/proto/hdfs.proto | 648 - .../hadoop-hdfs/src/main/proto/inotify.proto | 126 - .../hadoop-hdfs/src/main/proto/xattr.proto | 80 - .../src/main/resources/hdfs-default.xml | 68 +- .../src/main/webapps/hdfs/dfshealth.html | 87 +- .../src/main/webapps/hdfs/dfshealth.js | 71 +- .../webapps/static/dataTables.bootstrap.css | 371 + .../main/webapps/static/dataTables.bootstrap.js | 205 + .../src/main/webapps/static/hadoop.css | 51 + .../webapps/static/jquery.dataTables.min.js | 160 + .../src/main/webapps/static/moment.min.js | 7 + .../site/markdown/CentralizedCacheManagement.md | 2 + .../src/site/markdown/HDFSCommands.md | 34 +- .../markdown/HDFSHighAvailabilityWithNFS.md | 49 +- .../markdown/HDFSHighAvailabilityWithQJM.md | 42 +- .../src/site/markdown/HdfsImageViewer.md | 3 + .../src/site/markdown/HdfsMultihoming.md | 2 + .../src/site/markdown/HdfsNfsGateway.md | 2 +- .../src/site/markdown/MemoryStorage.md | 130 + .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 14 +- .../site/resources/images/LazyPersistWrites.png | Bin 0 -> 107161 bytes .../org/apache/hadoop/fs/TestGlobPaths.java | 25 + .../TestSWebHdfsFileContextMainOperations.java | 110 + .../TestWebHdfsFileContextMainOperations.java | 157 + .../contract/hdfs/TestHDFSContractAppend.java | 15 - .../hdfs/TestHDFSContractGetFileStatus.java | 46 + .../contract/hdfs/TestHDFSContractSetTimes.java | 45 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 39 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 494 +- .../apache/hadoop/hdfs/MiniDFSNNTopology.java | 18 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 227 +- .../hadoop/hdfs/TestDFSClientRetries.java | 66 +- .../hdfs/TestDFSInotifyEventInputStream.java | 47 +- .../apache/hadoop/hdfs/TestDFSInputStream.java | 25 + .../org/apache/hadoop/hdfs/TestDFSShell.java | 323 +- .../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 58 +- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 6 +- .../hadoop/hdfs/TestDatanodeLayoutUpgrade.java | 8 +- .../apache/hadoop/hdfs/TestDecommission.java | 124 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 108 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 6 + .../apache/hadoop/hdfs/TestFileCorruption.java | 19 +- .../apache/hadoop/hdfs/TestFileCreation.java | 6 - .../apache/hadoop/hdfs/TestLeaseRecovery.java | 46 + .../java/org/apache/hadoop/hdfs/TestQuota.java | 21 + .../org/apache/hadoop/hdfs/TestReplication.java | 26 +- .../apache/hadoop/hdfs/TestRollingUpgrade.java | 170 +- .../org/apache/hadoop/hdfs/TestSafeMode.java | 4 +- .../hdfs/protocol/TestBlockListAsLongs.java | 4 +- .../hadoop/hdfs/protocol/TestLayoutVersion.java | 102 +- .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 94 +- .../security/token/block/TestBlockToken.java | 10 +- .../hdfs/server/balancer/TestBalancer.java | 116 +- .../server/blockmanagement/TestBlockInfo.java | 14 +- .../TestBlockInfoUnderConstruction.java | 2 +- .../blockmanagement/TestBlockManager.java | 40 +- .../TestBlockReportRateLimiting.java | 220 + .../blockmanagement/TestBlockStatsMXBean.java | 145 + .../blockmanagement/TestCorruptReplicaInfo.java | 12 +- .../blockmanagement/TestDatanodeDescriptor.java | 4 +- .../blockmanagement/TestDatanodeManager.java | 21 +- .../blockmanagement/TestHeartbeatHandling.java | 6 +- .../TestNameNodePrunesMissingStorages.java | 2 +- .../blockmanagement/TestPendingReplication.java | 4 +- .../blockmanagement/TestReplicationPolicy.java | 10 +- .../hdfs/server/datanode/DataNodeTestUtils.java | 11 + .../server/datanode/SimulatedFSDataset.java | 5 + .../server/datanode/TestBPOfferService.java | 7 +- .../TestBlockHasMultipleReplicasOnSameDN.java | 2 +- .../hdfs/server/datanode/TestBlockRecovery.java | 6 +- .../datanode/TestBpServiceActorScheduler.java | 2 +- .../server/datanode/TestCachingStrategy.java | 5 +- .../TestDataXceiverLazyPersistHint.java | 178 + .../TestDatanodeProtocolRetryPolicy.java | 8 +- .../server/datanode/TestFsDatasetCache.java | 9 +- .../TestNNHandlesBlockReportPerStorage.java | 2 +- .../TestNNHandlesCombinedBlockReport.java | 2 +- .../datanode/TestReadOnlySharedStorage.java | 2 +- .../hdfs/server/datanode/TestStorageReport.java | 2 +- .../extdataset/ExternalReplicaInPipeline.java | 7 + .../fsdataset/impl/LazyPersistTestCase.java | 42 +- .../fsdataset/impl/TestFsDatasetImpl.java | 35 + .../impl/TestLazyPersistLockedMemory.java | 25 +- .../impl/TestLazyPersistReplicaPlacement.java | 36 +- .../datanode/fsdataset/impl/TestLazyWriter.java | 62 +- .../fsdataset/impl/TestRbwSpaceReservation.java | 73 + .../fsdataset/impl/TestScrLazyPersistFiles.java | 193 +- .../hadoop/hdfs/server/mover/TestMover.java | 55 +- .../hdfs/server/mover/TestStorageMover.java | 26 +- .../hdfs/server/namenode/CreateEditsLog.java | 4 +- .../hdfs/server/namenode/FSImageTestUtil.java | 2 +- .../hdfs/server/namenode/FSXAttrBaseTest.java | 5 +- .../server/namenode/NNThroughputBenchmark.java | 8 +- .../hdfs/server/namenode/NameNodeAdapter.java | 2 +- .../server/namenode/TestAclTransformation.java | 55 +- .../server/namenode/TestAddStripedBlocks.java | 20 +- .../hdfs/server/namenode/TestBackupNode.java | 2 +- ...stBlockPlacementPolicyRackFaultTolarent.java | 209 - ...stBlockPlacementPolicyRackFaultTolerant.java | 209 + .../namenode/TestBlockUnderConstruction.java | 4 +- .../server/namenode/TestCacheDirectives.java | 25 + .../hdfs/server/namenode/TestCheckpoint.java | 3 +- .../hdfs/server/namenode/TestClusterId.java | 6 +- .../TestCommitBlockSynchronization.java | 12 +- .../hdfs/server/namenode/TestDeadDatanode.java | 49 +- .../hdfs/server/namenode/TestEditLog.java | 16 +- .../server/namenode/TestFSEditLogLoader.java | 2 +- .../hdfs/server/namenode/TestFSImage.java | 50 +- .../hdfs/server/namenode/TestFSNamesystem.java | 48 + .../server/namenode/TestFSNamesystemMBean.java | 5 + .../hdfs/server/namenode/TestFileTruncate.java | 35 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 16 +- .../server/namenode/TestGetBlockLocations.java | 4 +- .../hdfs/server/namenode/TestHDFSConcat.java | 24 +- .../hdfs/server/namenode/TestINodeFile.java | 12 +- .../hdfs/server/namenode/TestLeaseManager.java | 65 +- .../server/namenode/TestNameNodeMXBean.java | 28 +- .../server/namenode/TestQuotaByStorageType.java | 14 +- .../namenode/TestQuotaWithStripedBlocks.java | 2 +- .../server/namenode/TestStripedINodeFile.java | 16 +- .../namenode/TestTruncateQuotaUpdate.java | 10 +- .../server/namenode/ha/HAStressTestHarness.java | 46 +- .../hdfs/server/namenode/ha/HATestUtil.java | 60 +- .../namenode/ha/TestBootstrapStandby.java | 176 +- .../ha/TestBootstrapStandbyWithQJM.java | 47 +- .../ha/TestDNFencingWithReplication.java | 1 + .../server/namenode/ha/TestEditLogTailer.java | 12 +- .../ha/TestFailoverWithBlockTokensEnabled.java | 55 +- .../server/namenode/ha/TestHAConfiguration.java | 49 +- .../namenode/ha/TestPipelinesFailover.java | 110 +- .../namenode/ha/TestRemoteNameNodeInfo.java | 61 + .../ha/TestRequestHedgingProxyProvider.java | 350 + .../namenode/ha/TestRetryCacheWithHA.java | 43 +- .../namenode/ha/TestSeveralNameNodes.java | 179 + .../namenode/ha/TestStandbyCheckpoints.java | 106 +- .../server/namenode/ha/TestStandbyIsHot.java | 2 + .../namenode/snapshot/SnapshotTestHelper.java | 9 +- .../snapshot/TestFileWithSnapshotFeature.java | 3 +- .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 117 +- .../hadoop/hdfs/tools/TestDebugAdmin.java | 8 + .../hdfs/tools/TestDelegationTokenFetcher.java | 54 + .../TestOfflineImageViewerForAcl.java | 24 + ...TestOfflineImageViewerForContentSummary.java | 248 + .../TestOfflineImageViewerForXAttr.java | 265 + .../hdfs/web/TestByteRangeInputStream.java | 35 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 100 + .../hdfs/web/TestWebHdfsFileSystemContract.java | 2 +- .../apache/hadoop/net/TestNetworkTopology.java | 1 + .../apache/hadoop/security/TestPermission.java | 26 + .../hadoop/test/MiniDFSClusterManager.java | 4 + .../org/apache/hadoop/tracing/TestTracing.java | 117 +- .../TestTracingShortCircuitLocalRead.java | 4 +- .../src/test/resources/contract/hdfs.xml | 10 + .../src/test/resources/hadoop-0.23-reserved.tgz | Bin 4558 -> 5590 bytes .../src/test/resources/hadoop-1-reserved.tgz | Bin 2572 -> 3348 bytes .../src/test/resources/hadoop-2-reserved.tgz | Bin 2838 -> 3465 bytes .../src/test/resources/hadoop-22-dfs-dir.tgz | Bin 318180 -> 413239 bytes .../src/test/resources/hadoop1-bbw.tgz | Bin 40234 -> 43294 bytes .../src/test/resources/log4j.properties | 2 +- .../src/test/resources/testHDFSConf.xml | 2 +- hadoop-mapreduce-project/CHANGES.txt | 153 +- hadoop-mapreduce-project/bin/mapred | 33 +- .../dev-support/findbugs-exclude.xml | 1 + .../hadoop/mapred/LocalContainerLauncher.java | 2 +- .../jobhistory/JobHistoryEventHandler.java | 27 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 30 +- .../v2/app/job/event/JobStartEvent.java | 2 +- .../event/JobTaskAttemptFetchFailureEvent.java | 9 +- .../TaskAttemptTooManyFetchFailureEvent.java | 50 + .../mapreduce/v2/app/job/impl/JobImpl.java | 53 +- .../v2/app/job/impl/TaskAttemptImpl.java | 25 +- .../mapreduce/v2/app/job/impl/TaskImpl.java | 4 +- .../mapreduce/v2/app/rm/RMCommunicator.java | 51 +- .../app/rm/RMContainerAllocationException.java | 31 + .../v2/app/rm/RMContainerAllocator.java | 23 +- .../v2/app/rm/RMContainerRequestor.java | 32 +- .../mapreduce/v2/app/webapp/AttemptsPage.java | 8 + .../mapreduce/v2/app/webapp/JobBlock.java | 2 + .../mapreduce/v2/app/webapp/TaskPage.java | 93 +- .../mapreduce/v2/app/webapp/dao/JobInfo.java | 6 + .../hadoop/mapreduce/jobhistory/TestEvents.java | 151 +- .../mapreduce/jobhistory/TestJobSummary.java | 10 +- .../v2/api/records/TestTaskAttemptReport.java | 131 + .../v2/api/records/TestTaskReport.java | 139 + .../apache/hadoop/mapreduce/v2/app/MRApp.java | 15 + .../mapreduce/v2/app/TestFetchFailure.java | 31 +- .../mapreduce/v2/app/TestMRAppMaster.java | 88 +- .../mapreduce/v2/app/job/impl/TestJobImpl.java | 130 +- .../v2/app/job/impl/TestTaskAttempt.java | 143 +- .../mapreduce/v2/app/rm/TestRMCommunicator.java | 99 + .../v2/app/rm/TestRMContainerAllocator.java | 137 +- .../v2/app/webapp/TestAMWebServicesJobs.java | 2 +- .../mapreduce/v2/app/webapp/TestBlocks.java | 92 +- .../v2/api/records/TaskAttemptReport.java | 3 + .../mapreduce/v2/api/records/TaskReport.java | 5 +- .../impl/pb/TaskAttemptReportPBImpl.java | 38 +- .../api/records/impl/pb/TaskReportPBImpl.java | 38 +- .../v2/jobhistory/FileNameIndexUtils.java | 17 +- .../mapreduce/v2/jobhistory/JHAdminConfig.java | 14 + .../apache/hadoop/mapreduce/v2/util/MRApps.java | 5 - .../v2/jobhistory/TestFileNameIndexUtils.java | 26 + .../apache/hadoop/mapred/FileInputFormat.java | 2 +- .../apache/hadoop/mapred/LineRecordReader.java | 4 +- .../java/org/apache/hadoop/mapred/Task.java | 4 +- .../org/apache/hadoop/mapred/lib/Chain.java | 2 +- .../hadoop/mapred/pipes/PipesPartitioner.java | 6 +- .../hadoop/mapreduce/JobResourceUploader.java | 38 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 20 + .../ClientDistributedCacheManager.java | 1 + .../mapreduce/jobhistory/AMStartedEvent.java | 24 +- .../mapreduce/jobhistory/AvroArrayUtils.java | 2 +- .../mapreduce/jobhistory/EventReader.java | 34 +- .../mapreduce/jobhistory/EventWriter.java | 70 +- .../mapreduce/jobhistory/JobFinishedEvent.java | 39 +- .../mapreduce/jobhistory/JobHistoryParser.java | 2 +- .../jobhistory/JobInfoChangeEvent.java | 12 +- .../mapreduce/jobhistory/JobInitedEvent.java | 24 +- .../jobhistory/JobPriorityChangeEvent.java | 10 +- .../jobhistory/JobStatusChangedEvent.java | 8 +- .../mapreduce/jobhistory/JobSubmittedEvent.java | 62 +- .../JobUnsuccessfulCompletionEvent.java | 4 +- .../jobhistory/MapAttemptFinishedEvent.java | 66 +- .../jobhistory/ReduceAttemptFinishedEvent.java | 70 +- .../jobhistory/TaskAttemptFinishedEvent.java | 34 +- .../jobhistory/TaskAttemptStartedEvent.java | 49 +- .../TaskAttemptUnsuccessfulCompletionEvent.java | 66 +- .../mapreduce/jobhistory/TaskFailedEvent.java | 32 +- .../mapreduce/jobhistory/TaskFinishedEvent.java | 26 +- .../mapreduce/jobhistory/TaskStartedEvent.java | 20 +- .../mapreduce/jobhistory/TaskUpdatedEvent.java | 10 +- .../mapreduce/lib/input/FileInputFormat.java | 3 +- .../mapreduce/lib/input/LineRecordReader.java | 3 +- .../lib/input/UncompressedSplitLineReader.java | 125 + .../lib/output/FileOutputCommitter.java | 2 +- .../hadoop/mapreduce/task/reduce/Fetcher.java | 1 + .../task/reduce/IFileWrappedMapOutput.java | 66 + .../task/reduce/InMemoryMapOutput.java | 26 +- .../mapreduce/task/reduce/LocalFetcher.java | 15 +- .../mapreduce/task/reduce/MergeManagerImpl.java | 5 +- .../mapreduce/task/reduce/OnDiskMapOutput.java | 33 +- .../task/reduce/ShuffleSchedulerImpl.java | 7 +- .../src/main/resources/mapred-default.xml | 76 +- .../src/site/markdown/EncryptedShuffle.md | 8 +- .../src/site/markdown/MapReduceTutorial.md | 2 +- .../src/site/markdown/MapredCommands.md | 14 +- .../hadoop/mapred/TestLineRecordReader.java | 77 +- .../TestClientDistributedCacheManager.java | 28 + .../lib/input/TestLineRecordReader.java | 79 +- .../mapreduce/task/reduce/TestFetcher.java | 27 +- .../hadoop/mapreduce/v2/hs/CompletedTask.java | 2 +- .../mapreduce/v2/hs/CompletedTaskAttempt.java | 2 +- .../mapreduce/v2/hs/JobHistoryServer.java | 14 +- .../mapreduce/v2/hs/server/HSAdminServer.java | 43 +- .../mapreduce/v2/hs/webapp/HsAttemptsPage.java | 7 + .../mapreduce/v2/hs/webapp/HsController.java | 28 +- .../mapreduce/v2/hs/webapp/HsCountersPage.java | 8 +- .../mapreduce/v2/hs/webapp/HsJobBlock.java | 2 +- .../mapreduce/v2/hs/webapp/HsLogsPage.java | 11 +- .../v2/hs/webapp/HsSingleCounterPage.java | 8 +- .../mapreduce/v2/hs/webapp/HsTaskPage.java | 11 +- .../hadoop/mapreduce/v2/hs/webapp/HsView.java | 18 +- .../v2/hs/server/TestHSAdminServer.java | 55 + .../mapreduce/v2/hs/webapp/TestBlocks.java | 2 +- .../org/apache/hadoop/mapred/YARNRunner.java | 30 +- .../java/org/apache/hadoop/hdfs/NNBench.java | 16 +- .../hadoop/mapred/TestLocalJobSubmission.java | 4 +- .../apache/hadoop/mapred/TestYARNRunner.java | 16 + .../mapred/pipes/TestPipeApplication.java | 5 +- .../hadoop-mapreduce-client-nativetask/pom.xml | 2 +- .../src/CMakeLists.txt | 337 +- .../src/JNIFlags.cmake | 118 - .../hadoop/mapred/FadvisedChunkedFile.java | 5 +- .../hadoop/mapred/FadvisedFileRegion.java | 6 +- .../apache/hadoop/mapred/ShuffleHandler.java | 18 +- .../hadoop/mapred/TestShuffleHandler.java | 105 + .../hadoop/examples/BaileyBorweinPlouffe.java | 2 +- .../apache/hadoop/examples/QuasiMonteCarlo.java | 4 + hadoop-mapreduce-project/pom.xml | 4 +- hadoop-maven-plugins/pom.xml | 8 + .../hadoop/maven/plugin/protoc/ProtocMojo.java | 188 +- hadoop-project-dist/pom.xml | 20 +- hadoop-project/pom.xml | 15 +- hadoop-project/src/site/site.xml | 3 +- .../org/apache/hadoop/tools/HadoopArchives.java | 108 +- .../src/site/markdown/HadoopArchives.md.vm | 2 +- .../apache/hadoop/tools/TestHadoopArchives.java | 26 +- hadoop-tools/hadoop-aws/pom.xml | 2 +- .../org/apache/hadoop/fs/s3/S3Credentials.java | 10 +- .../org/apache/hadoop/fs/s3a/Constants.java | 9 +- .../hadoop/fs/s3a/S3AFastOutputStream.java | 2 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 26 +- .../apache/hadoop/fs/s3a/S3AOutputStream.java | 15 +- .../src/site/markdown/tools/hadoop-aws/index.md | 6 + .../apache/hadoop/fs/s3/TestS3Credentials.java | 107 +- .../fs/azure/AzureNativeFileSystemStore.java | 15 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 13 +- .../hadoop/fs/azure/PageBlobInputStream.java | 32 +- .../hadoop/fs/azure/PageBlobOutputStream.java | 10 +- .../hadoop/fs/azure/StorageInterface.java | 6 +- .../hadoop/fs/azure/StorageInterfaceImpl.java | 4 +- .../hadoop/fs/azure/MockStorageInterface.java | 4 +- .../fs/azure/NativeAzureFileSystemBaseTest.java | 79 +- .../TestAzureFileSystemErrorConditions.java | 1 + .../hadoop/fs/azure/TestBlobDataValidation.java | 1 + ...tiveAzureFileSystemContractPageBlobLive.java | 90 + .../fs/azure/TestWasbUriAndConfiguration.java | 30 +- .../src/test/resources/azure-test.xml | 4 + .../java/org/apache/hadoop/tools/DistCp.java | 3 +- .../apache/hadoop/tools/DistCpConstants.java | 2 +- .../apache/hadoop/tools/DistCpOptionSwitch.java | 5 +- .../org/apache/hadoop/tools/DistCpOptions.java | 6 +- .../org/apache/hadoop/tools/OptionsParser.java | 2 +- .../apache/hadoop/tools/mapred/CopyMapper.java | 17 +- .../tools/mapred/RetriableFileCopyCommand.java | 2 +- .../apache/hadoop/tools/util/DistCpUtils.java | 41 +- .../hadoop/tools/util/ThrottledInputStream.java | 6 +- .../apache/hadoop/tools/TestOptionsParser.java | 16 +- .../gridmix/DummyResourceCalculatorPlugin.java | 42 + hadoop-tools/hadoop-openstack/pom.xml | 4 +- .../fs/swift/snative/SwiftNativeFileSystem.java | 9 + .../fs/swift/TestSwiftFileSystemBasicOps.java | 7 + hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 49 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 8 +- .../yarn/sls/scheduler/RMNodeWrapper.java | 5 + .../sls/scheduler/ResourceSchedulerWrapper.java | 10 + .../apache/hadoop/record/BinaryRecordInput.java | 7 +- .../hadoop/record/BinaryRecordOutput.java | 17 +- .../org/apache/hadoop/streaming/PipeMapRed.java | 19 +- .../org/apache/hadoop/streaming/StreamJob.java | 5 +- .../hadoop/typedbytes/TypedBytesInput.java | 8 +- .../hadoop/typedbytes/TypedBytesOutput.java | 8 +- .../typedbytes/TypedBytesRecordInput.java | 8 +- .../typedbytes/TypedBytesRecordOutput.java | 8 +- .../typedbytes/TypedBytesWritableInput.java | 8 +- .../typedbytes/TypedBytesWritableOutput.java | 8 +- .../src/site/markdown/HadoopStreaming.md.vm | 9 + hadoop-yarn-project/CHANGES.txt | 444 +- .../hadoop-yarn/bin/start-yarn.sh | 7 +- .../hadoop-yarn/bin/stop-yarn.sh | 7 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 61 +- .../hadoop-yarn/hadoop-yarn-api/pom.xml | 34 + .../hadoop/yarn/api/ApplicationConstants.java | 58 +- .../yarn/api/records/ApplicationReport.java | 48 + .../hadoop/yarn/api/records/NodeState.java | 10 +- .../yarn/api/records/QueueStatistics.java | 36 + .../hadoop/yarn/conf/YarnConfiguration.java | 112 +- .../yarn/server/api/ContainerContext.java | 19 + .../api/ContainerInitializationContext.java | 7 + .../server/api/ContainerTerminationContext.java | 7 + .../hadoop/yarn/server/api/ContainerType.java | 34 + .../apache/hadoop/yarn/util/package-info.java | 2 - .../src/main/proto/yarn_protos.proto | 11 + .../yarn/conf/TestYarnConfigurationFields.java | 136 + .../distributedshell/ApplicationMaster.java | 31 +- .../distributedshell/TestDSAppMaster.java | 11 +- .../org/apache/hadoop/yarn/client/SCMAdmin.java | 2 +- .../api/async/impl/NMClientAsyncImpl.java | 4 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 6 +- .../hadoop/yarn/client/cli/ClusterCLI.java | 2 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 32 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 24 +- .../hadoop/yarn/client/ProtocolHATestBase.java | 26 +- ...estApplicationMasterServiceProtocolOnHA.java | 10 +- .../hadoop/yarn/client/TestRMFailover.java | 26 +- .../yarn/client/TestResourceTrackerOnHA.java | 2 +- .../yarn/client/api/impl/TestYarnClient.java | 10 +- .../hadoop/yarn/client/cli/TestClusterCLI.java | 4 +- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 17 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 5 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +- .../impl/pb/ApplicationReportPBImpl.java | 49 + .../yarn/api/records/impl/pb/ProtoUtils.java | 12 + .../api/records/impl/pb/QueueInfoPBImpl.java | 2 +- .../records/impl/pb/QueueStatisticsPBImpl.java | 36 + .../records/impl/pb/ResourceRequestPBImpl.java | 5 +- .../impl/pb/SerializedExceptionPBImpl.java | 28 +- .../apache/hadoop/yarn/client/ServerProxy.java | 23 +- .../client/api/impl/TimelineClientImpl.java | 28 +- .../yarn/client/api/impl/package-info.java | 4 - .../hadoop/yarn/client/api/package-info.java | 4 - .../hadoop/yarn/event/AsyncDispatcher.java | 17 +- .../hadoop/yarn/factories/package-info.java | 2 - .../yarn/factory/providers/package-info.java | 2 - .../logaggregation/AggregatedLogFormat.java | 83 +- .../nodelabels/CommonNodeLabelsManager.java | 37 +- .../nodelabels/FileSystemNodeLabelsStore.java | 47 +- .../yarn/security/ContainerTokenIdentifier.java | 43 +- .../state/InvalidStateTransitionException.java | 51 + .../state/InvalidStateTransitonException.java | 21 +- .../apache/hadoop/yarn/state/StateMachine.java | 2 +- .../hadoop/yarn/state/StateMachineFactory.java | 10 +- .../java/org/apache/hadoop/yarn/util/Apps.java | 19 +- .../apache/hadoop/yarn/util/CpuTimeTracker.java | 100 - .../util/LinuxResourceCalculatorPlugin.java | 352 +- .../yarn/util/ProcfsBasedProcessTree.java | 34 +- .../org/apache/hadoop/yarn/util/RMHAUtils.java | 33 +- .../yarn/util/ResourceCalculatorPlugin.java | 116 +- .../yarn/util/WindowsBasedProcessTree.java | 2 +- .../util/WindowsResourceCalculatorPlugin.java | 152 +- .../resource/DominantResourceCalculator.java | 15 + .../hadoop/yarn/util/resource/Resources.java | 14 +- .../hadoop/yarn/webapp/YarnWebParams.java | 3 + .../yarn/webapp/view/TwoColumnLayout.java | 2 +- .../main/proto/server/yarn_security_token.proto | 69 - .../src/main/proto/yarn_security_token.proto | 70 + .../resources/webapps/static/yarn.dt.plugins.js | 16 +- .../src/main/resources/yarn-default.xml | 600 +- .../hadoop/yarn/api/TestApplicatonReport.java | 4 +- .../impl/pb/TestSerializedExceptionPBImpl.java | 23 +- .../hadoop/yarn/conf/TestYarnConfiguration.java | 22 + .../yarn/conf/TestYarnConfigurationFields.java | 106 - .../hadoop/yarn/event/DrainDispatcher.java | 11 +- .../hadoop/yarn/event/TestAsyncDispatcher.java | 79 + .../nodelabels/TestCommonNodeLabelsManager.java | 17 +- .../yarn/security/TestYARNTokenIdentifier.java | 53 + .../org/apache/hadoop/yarn/util/TestApps.java | 61 + .../util/TestLinuxResourceCalculatorPlugin.java | 235 - .../util/TestResourceCalculatorProcessTree.java | 2 +- .../hadoop/yarn/util/TestWebAppUtils.java | 81 - .../TestWindowsResourceCalculatorPlugin.java | 86 - .../util/resource/TestResourceCalculator.java | 125 + .../yarn/webapp/util/TestWebAppUtils.java | 52 + .../pom.xml | 2 +- .../ApplicationHistoryClientService.java | 9 +- .../ApplicationHistoryManager.java | 16 +- .../ApplicationHistoryManagerImpl.java | 4 +- ...pplicationHistoryManagerOnTimelineStore.java | 63 +- .../ApplicationHistoryServer.java | 14 +- .../webapp/AHSView.java | 2 +- .../server/timeline/TimelineDataManager.java | 123 + .../timeline/TimelineDataManagerMetrics.java | 174 + ...TimelineAuthenticationFilterInitializer.java | 5 +- .../TestApplicationHistoryClientService.java | 97 +- ...pplicationHistoryManagerOnTimelineStore.java | 34 +- .../webapp/TestAHSWebServices.java | 64 +- .../timeline/TestTimelineDataManager.java | 1 + .../TestTimelineAuthenticationFilter.java | 11 + .../hadoop/yarn/server/api/ResourceTracker.java | 16 +- .../pb/client/ResourceTrackerPBClientImpl.java | 18 + .../service/ResourceTrackerPBServiceImpl.java | 27 +- .../UnRegisterNodeManagerRequest.java | 38 + .../UnRegisterNodeManagerResponse.java | 30 + .../pb/UnRegisterNodeManagerRequestPBImpl.java | 108 + .../pb/UnRegisterNodeManagerResponsePBImpl.java | 70 + .../yarn/server/api/records/NodeStatus.java | 59 +- .../server/api/records/ResourceUtilization.java | 133 + .../api/records/impl/pb/NodeStatusPBImpl.java | 56 +- .../impl/pb/ResourceUtilizationPBImpl.java | 104 + .../yarn/server/api/records/package-info.java | 19 + .../metrics/ApplicationMetricsConstants.java | 7 + .../hadoop/yarn/server/utils/BuilderUtils.java | 4 +- .../server/utils/YarnServerBuilderUtils.java | 11 +- .../hadoop/yarn/server/webapp/AppBlock.java | 24 +- .../hadoop/yarn/server/webapp/AppsBlock.java | 49 +- .../hadoop/yarn/server/webapp/WebPageUtils.java | 10 +- .../hadoop/yarn/server/webapp/WebServices.java | 53 +- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 35 +- .../src/main/proto/ResourceTracker.proto | 1 + .../main/proto/yarn_server_common_protos.proto | 8 + .../yarn_server_common_service_protos.proto | 7 + .../yarn/TestResourceTrackerPBClientImpl.java | 34 +- .../apache/hadoop/yarn/TestYSCRPCFactories.java | 10 +- .../hadoop/yarn/TestYarnServerApiClasses.java | 12 + .../hadoop-yarn-server-nodemanager/pom.xml | 2 +- .../src/CMakeLists.txt | 23 +- .../server/nodemanager/ContainerExecutor.java | 54 +- .../hadoop/yarn/server/nodemanager/Context.java | 2 + .../nodemanager/DefaultContainerExecutor.java | 2 +- .../nodemanager/DockerContainerExecutor.java | 2 +- .../nodemanager/LinuxContainerExecutor.java | 226 +- .../nodemanager/LocalDirsHandlerService.java | 59 +- .../yarn/server/nodemanager/NodeManager.java | 51 +- .../server/nodemanager/NodeResourceMonitor.java | 10 +- .../nodemanager/NodeResourceMonitorImpl.java | 140 + .../nodemanager/NodeStatusUpdaterImpl.java | 83 +- .../containermanager/AuxServices.java | 6 +- .../application/ApplicationImpl.java | 4 +- .../container/ContainerImpl.java | 11 +- .../launcher/ContainerLaunch.java | 24 +- .../launcher/RecoveredContainerLaunch.java | 3 +- .../linux/privileged/PrivilegedOperation.java | 47 +- .../PrivilegedOperationException.java | 30 +- .../privileged/PrivilegedOperationExecutor.java | 32 +- .../linux/resources/CGroupsHandler.java | 8 + .../linux/resources/CGroupsHandlerImpl.java | 12 +- .../runtime/DefaultLinuxContainerRuntime.java | 148 + .../DelegatingLinuxContainerRuntime.java | 110 + .../runtime/DockerLinuxContainerRuntime.java | 273 + .../linux/runtime/LinuxContainerRuntime.java | 38 + .../runtime/LinuxContainerRuntimeConstants.java | 69 + .../linux/runtime/docker/DockerClient.java | 82 + .../linux/runtime/docker/DockerCommand.java | 66 + .../linux/runtime/docker/DockerLoadCommand.java | 30 + .../linux/runtime/docker/DockerRunCommand.java | 107 + .../localizer/LocalizedResource.java | 4 +- .../localizer/ResourceLocalizationService.java | 31 +- .../logaggregation/AppLogAggregatorImpl.java | 10 +- .../monitor/ContainersMonitor.java | 3 +- .../monitor/ContainersMonitorImpl.java | 47 +- .../runtime/ContainerExecutionException.java | 85 + .../runtime/ContainerRuntime.java | 50 + .../runtime/ContainerRuntimeConstants.java | 33 + .../runtime/ContainerRuntimeContext.java | 105 + .../executor/ContainerLivenessContext.java | 13 + .../executor/ContainerReacquisitionContext.java | 13 + .../executor/ContainerSignalContext.java | 13 + .../executor/ContainerStartContext.java | 23 +- .../nodemanager/metrics/NodeManagerMetrics.java | 13 +- .../util/CgroupsLCEResourcesHandler.java | 19 +- .../util/NodeManagerHardwareUtils.java | 238 +- .../nodemanager/webapp/ContainerLogsUtils.java | 2 +- .../server/nodemanager/webapp/NodePage.java | 8 +- .../server/nodemanager/webapp/dao/NodeInfo.java | 6 + .../container-executor/impl/configuration.c | 30 +- .../container-executor/impl/configuration.h | 2 + .../impl/container-executor.c | 466 +- .../impl/container-executor.h | 29 +- .../main/native/container-executor/impl/main.c | 97 +- .../test/test-container-executor.c | 56 +- .../server/nodemanager/LocalRMInterface.java | 10 + .../nodemanager/MockNodeStatusUpdater.java | 9 + .../nodemanager/TestContainerExecutor.java | 56 +- .../TestLinuxContainerExecutorWithMocks.java | 131 +- .../TestLocalDirsHandlerService.java | 2 +- .../nodemanager/TestNodeResourceMonitor.java | 35 + .../nodemanager/TestNodeStatusUpdater.java | 78 + .../TestNodeStatusUpdaterForLabels.java | 8 + .../containermanager/TestNMProxy.java | 99 +- .../launcher/TestContainerLaunch.java | 10 +- .../TestPrivilegedOperationExecutor.java | 14 +- .../runtime/TestDockerContainerRuntime.java | 219 + .../TestResourceLocalizationService.java | 10 +- .../TestLogAggregationService.java | 90 +- .../monitor/TestContainersMonitor.java | 2 +- .../util/TestCgroupsLCEResourcesHandler.java | 2 + .../util/TestNodeManagerHardwareUtils.java | 148 +- .../webapp/TestContainerLogsPage.java | 70 +- .../nodemanager/webapp/TestNMWebServices.java | 2 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 14 +- .../server/resourcemanager/AdminService.java | 19 +- .../ApplicationMasterService.java | 19 +- .../server/resourcemanager/ClientRMService.java | 11 +- .../server/resourcemanager/ClusterMetrics.java | 14 + .../resourcemanager/NodesListManager.java | 28 +- .../resourcemanager/RMActiveServiceContext.java | 30 - .../server/resourcemanager/RMAppManager.java | 20 +- .../server/resourcemanager/RMContextImpl.java | 26 +- .../server/resourcemanager/ResourceManager.java | 79 +- .../resourcemanager/ResourceTrackerService.java | 46 +- .../resourcemanager/amlauncher/AMLauncher.java | 29 +- .../amlauncher/ApplicationMasterLauncher.java | 30 +- .../metrics/ApplicationCreatedEvent.java | 25 +- .../metrics/SystemMetricsPublisher.java | 11 +- .../monitor/SchedulingEditPolicy.java | 6 +- .../monitor/SchedulingMonitor.java | 3 +- .../ProportionalCapacityPreemptionPolicy.java | 63 +- .../nodelabels/RMNodeLabelsManager.java | 13 +- .../recovery/FileSystemRMStateStore.java | 229 +- .../recovery/LeveldbRMStateStore.java | 114 +- .../recovery/MemoryRMStateStore.java | 57 + .../recovery/NullRMStateStore.java | 23 + .../resourcemanager/recovery/RMStateStore.java | 171 +- .../recovery/RMStateStoreEventType.java | 5 +- .../RMStateStoreStoreReservationEvent.java | 56 + .../recovery/ZKRMStateStore.java | 881 +- .../reservation/AbstractReservationSystem.java | 2 + .../reservation/GreedyReservationAgent.java | 385 - .../reservation/InMemoryPlan.java | 22 +- .../InMemoryReservationAllocation.java | 30 +- .../resourcemanager/reservation/Plan.java | 2 +- .../reservation/PlanContext.java | 2 + .../resourcemanager/reservation/PlanView.java | 40 +- .../resourcemanager/reservation/Planner.java | 47 - .../RLESparseResourceAllocation.java | 98 +- .../reservation/ReservationAgent.java | 72 - .../reservation/ReservationAllocation.java | 3 +- .../ReservationSchedulerConfiguration.java | 16 +- .../reservation/ReservationSystem.java | 14 +- .../reservation/ReservationSystemUtil.java | 155 + .../reservation/SimpleCapacityReplanner.java | 113 - .../planning/AlignedPlannerWithGreedy.java | 123 + .../planning/GreedyReservationAgent.java | 97 + .../reservation/planning/IterativePlanner.java | 338 + .../reservation/planning/Planner.java | 49 + .../reservation/planning/PlanningAlgorithm.java | 207 + .../reservation/planning/ReservationAgent.java | 73 + .../planning/SimpleCapacityReplanner.java | 118 + .../reservation/planning/StageAllocator.java | 55 + .../planning/StageAllocatorGreedy.java | 152 + .../planning/StageAllocatorLowCostAligned.java | 360 + .../planning/StageEarliestStart.java | 46 + .../planning/StageEarliestStartByDemand.java | 106 + .../StageEarliestStartByJobArrival.java | 39 + .../planning/TryManyReservationAgents.java | 114 + .../server/resourcemanager/rmapp/RMAppImpl.java | 44 +- .../rmapp/attempt/RMAppAttemptImpl.java | 22 +- .../rmcontainer/RMContainerImpl.java | 76 +- .../server/resourcemanager/rmnode/RMNode.java | 2 + .../resourcemanager/rmnode/RMNodeEventType.java | 1 + .../resourcemanager/rmnode/RMNodeImpl.java | 86 +- .../scheduler/AbstractYarnScheduler.java | 61 +- .../scheduler/AppSchedulingInfo.java | 7 +- .../scheduler/ContainerPreemptEvent.java | 8 +- .../scheduler/ContainerPreemptEventType.java | 26 - .../server/resourcemanager/scheduler/Queue.java | 8 + .../scheduler/ResourceLimits.java | 19 +- .../scheduler/SchedulerApplication.java | 22 + .../scheduler/SchedulerApplicationAttempt.java | 39 +- .../scheduler/YarnScheduler.java | 31 + .../scheduler/capacity/AbstractCSQueue.java | 45 +- .../scheduler/capacity/CSAssignment.java | 23 +- .../capacity/CapacityHeadroomProvider.java | 16 +- .../scheduler/capacity/CapacityScheduler.java | 246 +- .../CapacitySchedulerConfiguration.java | 20 +- .../capacity/CapacitySchedulerContext.java | 2 - .../scheduler/capacity/LeafQueue.java | 959 +- .../scheduler/capacity/ParentQueue.java | 36 +- .../scheduler/capacity/ReservationQueue.java | 4 - .../capacity/allocator/AllocationState.java | 28 + .../capacity/allocator/ContainerAllocation.java | 89 + .../capacity/allocator/ContainerAllocator.java | 150 + .../allocator/RegularContainerAllocator.java | 698 + .../scheduler/common/fica/FiCaSchedulerApp.java | 132 +- .../scheduler/event/AppAddedSchedulerEvent.java | 28 +- .../event/ContainerRescheduledEvent.java | 35 + .../scheduler/event/SchedulerEventType.java | 10 +- .../scheduler/fair/FSAppAttempt.java | 132 +- .../scheduler/fair/FSLeafQueue.java | 52 +- .../scheduler/fair/FSOpDurations.java | 6 + .../scheduler/fair/FSParentQueue.java | 219 +- .../resourcemanager/scheduler/fair/FSQueue.java | 21 + .../scheduler/fair/FSSchedulerNode.java | 11 +- .../scheduler/fair/FairScheduler.java | 127 +- .../scheduler/fair/QueueManager.java | 3 +- .../scheduler/fair/SchedulingPolicy.java | 11 + .../DominantResourceFairnessPolicy.java | 18 +- .../fair/policies/FairSharePolicy.java | 11 +- .../scheduler/fair/policies/FifoPolicy.java | 15 +- .../scheduler/fifo/FifoScheduler.java | 15 + .../AbstractComparatorOrderingPolicy.java | 31 + .../scheduler/policy/FairOrderingPolicy.java | 11 +- .../scheduler/policy/FifoComparator.java | 11 +- .../scheduler/policy/FifoOrderingPolicy.java | 6 +- .../scheduler/policy/OrderingPolicy.java | 5 + .../scheduler/policy/SchedulableEntity.java | 5 + .../security/RMContainerTokenSecretManager.java | 13 +- .../webapp/CapacitySchedulerPage.java | 12 +- .../webapp/DefaultSchedulerPage.java | 4 +- .../webapp/FairSchedulerAppsBlock.java | 6 + .../webapp/FairSchedulerPage.java | 10 +- .../webapp/MetricsOverviewTable.java | 2 + .../resourcemanager/webapp/NodesPage.java | 1 + .../resourcemanager/webapp/RMAppsBlock.java | 25 +- .../resourcemanager/webapp/RMWebServices.java | 27 +- .../resourcemanager/webapp/dao/AppInfo.java | 19 +- .../webapp/dao/CapacitySchedulerQueueInfo.java | 18 + .../webapp/dao/ClusterMetricsInfo.java | 8 +- .../webapp/dao/FairSchedulerQueueInfo.java | 28 +- .../yarn_server_resourcemanager_recovery.proto | 18 +- .../server/resourcemanager/Application.java | 12 + .../yarn/server/resourcemanager/HATestUtil.java | 38 + .../yarn/server/resourcemanager/MockAM.java | 14 +- .../yarn/server/resourcemanager/MockNodes.java | 5 + .../yarn/server/resourcemanager/MockRM.java | 57 +- .../server/resourcemanager/TestAppManager.java | 6 +- .../TestApplicationMasterLauncher.java | 7 +- .../resourcemanager/TestClientRMService.java | 1 + .../resourcemanager/TestRMAdminService.java | 49 +- .../resourcemanager/TestRMDispatcher.java | 79 + .../resourcemanager/TestRMEmbeddedElector.java | 28 +- .../resourcemanager/TestRMNodeTransitions.java | 112 +- .../server/resourcemanager/TestRMRestart.java | 8 +- .../resourcemanager/TestResourceManager.java | 1 + .../TestResourceTrackerService.java | 123 +- .../TestWorkPreservingRMRestart.java | 10 +- .../applicationsmanager/TestAMRestart.java | 2 +- .../TestRMAppLogAggregationStatus.java | 4 +- .../metrics/TestSystemMetricsPublisher.java | 53 + ...estProportionalCapacityPreemptionPolicy.java | 308 +- ...pacityPreemptionPolicyForNodePartitions.java | 135 +- .../nodelabels/TestRMNodeLabelsManager.java | 48 +- .../recovery/RMStateStoreTestBase.java | 189 +- .../recovery/TestFSRMStateStore.java | 22 +- .../recovery/TestLeveldbRMStateStore.java | 6 + .../resourcemanager/recovery/TestProtos.java | 36 + .../recovery/TestZKRMStateStore.java | 84 +- .../recovery/TestZKRMStateStorePerf.java | 12 +- .../TestZKRMStateStoreZKClientConnections.java | 181 +- .../reservation/ReservationSystemTestUtil.java | 197 +- .../reservation/TestCapacityOverTimePolicy.java | 18 +- .../TestCapacityReservationSystem.java | 94 - .../TestCapacitySchedulerPlanFollower.java | 12 +- .../reservation/TestFairReservationSystem.java | 128 - .../TestFairSchedulerPlanFollower.java | 72 +- .../reservation/TestGreedyReservationAgent.java | 604 - .../reservation/TestInMemoryPlan.java | 39 +- .../TestInMemoryReservationAllocation.java | 65 +- .../reservation/TestNoOverCommitPolicy.java | 1 + .../TestRLESparseResourceAllocation.java | 84 +- .../reservation/TestReservationSystem.java | 213 + .../TestSchedulerPlanFollowerBase.java | 1 + .../TestSimpleCapacityReplanner.java | 161 - .../planning/TestAlignedPlanner.java | 820 + .../planning/TestGreedyReservationAgent.java | 611 + .../planning/TestSimpleCapacityReplanner.java | 170 + .../resourcetracker/TestNMReconnect.java | 67 +- .../rmapp/TestNodesListManager.java | 162 + .../rmapp/TestRMAppTransitions.java | 6 +- .../attempt/TestRMAppAttemptTransitions.java | 8 +- .../rmcontainer/TestRMContainerImpl.java | 79 +- .../scheduler/TestAbstractYarnScheduler.java | 164 +- .../scheduler/TestSchedulerUtils.java | 47 + .../capacity/TestApplicationLimits.java | 34 +- .../capacity/TestApplicationPriority.java | 570 + .../capacity/TestCapacityScheduler.java | 276 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 64 + .../scheduler/capacity/TestChildQueueOrder.java | 3 +- .../capacity/TestContainerAllocation.java | 90 +- .../scheduler/capacity/TestLeafQueue.java | 547 +- .../scheduler/capacity/TestParentQueue.java | 2 - .../scheduler/capacity/TestQueueParsing.java | 71 + .../capacity/TestReservationQueue.java | 26 +- .../scheduler/capacity/TestReservations.java | 119 +- .../scheduler/capacity/TestUtils.java | 30 +- .../scheduler/fair/FairSchedulerTestBase.java | 31 +- .../scheduler/fair/TestFSLeafQueue.java | 64 + .../scheduler/fair/TestFairScheduler.java | 493 +- .../scheduler/fair/TestSchedulingUpdate.java | 135 + .../scheduler/fifo/TestFifoScheduler.java | 23 +- .../scheduler/policy/MockSchedulableEntity.java | 13 +- .../security/TestDelegationTokenRenewer.java | 29 +- .../resourcemanager/webapp/TestNodesPage.java | 4 +- .../webapp/TestRMWebServices.java | 21 +- .../webapp/TestRMWebServicesApps.java | 36 +- .../TestRMWebServicesAppsModification.java | 2 +- .../webapp/TestRMWebServicesCapacitySched.java | 4 +- .../webapp/TestRMWebServicesNodeLabels.java | 51 +- .../webapp/dao/TestFairSchedulerQueueInfo.java | 59 + .../hadoop/yarn/server/MiniYARNCluster.java | 32 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 37 + .../yarn/server/webproxy/AppReportFetcher.java | 79 +- .../yarn/server/webproxy/WebAppProxyServer.java | 32 +- .../server/webproxy/WebAppProxyServlet.java | 5 +- .../server/webproxy/TestAppReportFetcher.java | 117 + .../server/webproxy/TestWebAppProxyServlet.java | 57 + .../amfilter/TestAmFilterInitializer.java | 47 + .../src/site/markdown/NodeManager.md | 49 +- .../src/site/markdown/NodeManagerRestart.md | 53 - .../src/site/markdown/ResourceManagerHA.md | 16 +- .../src/site/markdown/ResourceManagerRest.md | 20 +- .../src/site/markdown/ResourceManagerRestart.md | 32 +- .../src/site/markdown/TimelineServer.md | 77 +- .../hadoop-yarn-site/src/site/markdown/YARN.md | 22 +- .../src/site/markdown/YarnCommands.md | 10 +- hadoop-yarn-project/pom.xml | 2 +- pom.xml | 2 + 1193 files changed, 53822 insertions(+), 50194 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b6a63bb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b6a63bb/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index 33c2ed9,aeaa980..03b4a1c --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@@ -53,6 -63,48 +63,49 @@@ http://maven.apache.org/xsd/maven-4.0.0 </excludes> </configuration> </plugin> + <plugin> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> + <executions> + <execution> + <id>compile-protoc</id> + <phase>generate-sources</phase> + <goals> + <goal>protoc</goal> + </goals> + <configuration> + <protocVersion>${protobuf.version}</protocVersion> + <protocCommand>${protoc.path}</protocCommand> + <imports> + <param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param> + <param>${basedir}/src/main/proto</param> + </imports> + <source> + <directory>${basedir}/src/main/proto</directory> + <includes> + <include>ClientDatanodeProtocol.proto</include> + <include>ClientNamenodeProtocol.proto</include> + <include>acl.proto</include> + <include>xattr.proto</include> + <include>datatransfer.proto</include> + <include>hdfs.proto</include> + <include>encryption.proto</include> + <include>inotify.proto</include> ++ <include>erasurecoding.proto</include> + </includes> + </source> + <output>${project.build.directory}/generated-sources/java</output> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <configuration> + <excludePackageNames>org.apache.hadoop.hdfs.protocol.proto</excludePackageNames> + </configuration> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b6a63bb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java ----------------------------------------------------------------------
