Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcc8f65a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcc8f65a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcc8f65a Branch: refs/heads/HADOOP-13345 Commit: fcc8f65a0ba0e382c0fc202fcefd4141531c95fb Parents: b54e1b2 49efd5d Author: Mingliang Liu <[email protected]> Authored: Mon Mar 20 16:17:51 2017 -0700 Committer: Mingliang Liu <[email protected]> Committed: Mon Mar 20 16:17:51 2017 -0700 ---------------------------------------------------------------------- LICENSE.txt | 22 +- hadoop-common-project/hadoop-common/pom.xml | 5 - .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 14 + .../org/apache/hadoop/fs/viewfs/ViewFs.java | 77 +- .../hadoop/io/retry/RetryInvocationHandler.java | 17 +- .../org/apache/hadoop/net/InnerNodeImpl.java | 8 +- .../net/NetworkTopologyWithNodeGroup.java | 2 +- .../org/apache/hadoop/cli/CLITestHelper.java | 17 +- .../apache/hadoop/cli/util/CommandExecutor.java | 4 +- .../apache/hadoop/cli/util/FSCmdExecutor.java | 4 +- .../hadoop/fs/FileSystemContractBaseTest.java | 246 ++-- .../fs/TestRawLocalFileSystemContract.java | 24 +- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 92 +- .../apache/hadoop/oncrpc/SimpleTcpServer.java | 1 + .../java/org/apache/hadoop/portmap/Portmap.java | 1 + hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 - .../java/org/apache/hadoop/hdfs/DFSClient.java | 10 +- .../apache/hadoop/hdfs/FileChecksumHelper.java | 42 +- .../dev-support/findbugsExcludeFile.xml | 12 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../hadoop/hdfs/net/DFSNetworkTopology.java | 289 ++++ .../hadoop/hdfs/net/DFSTopologyNodeImpl.java | 302 ++++ .../server/blockmanagement/BlockManager.java | 2 +- .../blockmanagement/DatanodeDescriptor.java | 9 + .../server/datanode/BlockChecksumHelper.java | 6 +- .../hdfs/server/datanode/BlockReceiver.java | 4 + .../server/datanode/BlockRecoveryWorker.java | 10 +- .../hadoop/hdfs/server/datanode/DataNode.java | 14 + .../hdfs/server/datanode/DataNodeMXBean.java | 7 + .../server/datanode/checker/AbstractFuture.java | 1295 ++++++++++++++++++ .../datanode/checker/DatasetVolumeChecker.java | 15 +- .../checker/StorageLocationChecker.java | 1 + .../datanode/checker/ThrottledAsyncChecker.java | 37 +- .../server/datanode/checker/TimeoutFuture.java | 162 +++ .../datanode/metrics/DataNodeDiskMetrics.java | 13 +- .../namenode/ErasureCodingPolicyManager.java | 5 +- .../hdfs/server/namenode/FSNamesystem.java | 7 +- .../ha/RequestHedgingProxyProvider.java | 41 +- .../apache/hadoop/hdfs/tools/AdminHelper.java | 6 +- .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 2 +- .../offlineEditsViewer/OfflineEditsViewer.java | 19 +- .../src/main/resources/hdfs-default.xml | 5 +- .../src/site/markdown/HDFSErasureCoding.md | 13 +- .../src/site/markdown/HdfsEditsViewer.md | 2 + .../hadoop/cli/util/CacheAdminCmdExecutor.java | 4 +- .../hadoop/cli/util/CryptoAdminCmdExecutor.java | 4 +- .../cli/util/ErasureCodingCliCmdExecutor.java | 4 +- .../apache/hadoop/hdfs/DFSNetworkTopology.java | 36 - .../apache/hadoop/hdfs/DFSTopologyNodeImpl.java | 253 ---- .../hadoop/hdfs/TestDFSNetworkTopology.java | 260 ---- .../hdfs/TestDecommissionWithStriped.java | 2 + .../hadoop/hdfs/TestDistributedFileSystem.java | 22 +- .../TestErasureCodeBenchmarkThroughput.java | 2 + .../TestErasureCodingPolicyWithSnapshot.java | 2 + .../apache/hadoop/hdfs/TestFileChecksum.java | 2 + .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 5 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 + .../hdfs/TestReadStripedFileWithDecoding.java | 2 + .../TestReadStripedFileWithMissingBlocks.java | 2 + .../hadoop/hdfs/TestReconstructStripedFile.java | 2 + .../org/apache/hadoop/hdfs/TestSafeMode.java | 24 + .../hdfs/TestSafeModeWithStripedFile.java | 2 + .../hadoop/hdfs/TestWriteReadStripedFile.java | 2 + .../hadoop/hdfs/net/TestDFSNetworkTopology.java | 511 +++++++ .../hdfs/server/balancer/TestBalancer.java | 13 +- .../blockmanagement/TestBlockManager.java | 113 +- .../TestBlockTokenWithDFSStriped.java | 4 + ...constructStripedBlocksWithRackAwareness.java | 6 + .../TestSequentialBlockGroupId.java | 2 + .../TestDataNodeErasureCodingMetrics.java | 2 + .../server/datanode/TestDataNodeMXBean.java | 33 +- .../TestDatasetVolumeCheckerTimeout.java | 134 ++ .../checker/TestThrottledAsyncChecker.java | 14 +- .../TestThrottledAsyncCheckerTimeout.java | 223 +++ .../hadoop/hdfs/server/mover/TestMover.java | 2 + .../TestAddOverReplicatedStripedBlocks.java | 4 +- .../namenode/TestAddStripedBlockInFBR.java | 3 + .../server/namenode/TestAddStripedBlocks.java | 10 +- .../server/namenode/TestEnabledECPolicies.java | 15 +- .../server/namenode/TestFSEditLogLoader.java | 4 + .../hadoop/hdfs/server/namenode/TestFsck.java | 8 + .../server/namenode/TestNameNodeMXBean.java | 2 + .../namenode/TestQuotaWithStripedBlocks.java | 2 + .../namenode/TestReconstructStripedBlocks.java | 6 + .../server/namenode/TestStripedINodeFile.java | 4 + .../ha/TestRequestHedgingProxyProvider.java | 108 +- .../TestOfflineEditsViewer.java | 18 + ...TestOfflineImageViewerWithStripedBlocks.java | 2 + .../src/test/resources/testCryptoConf.xml | 31 + .../test/resources/testErasureCodingConf.xml | 16 + .../hadoop/filecache/DistributedCache.java | 8 +- .../mapreduce/filecache/DistributedCache.java | 8 +- hadoop-project/pom.xml | 5 + .../oss/TestAliyunOSSFileSystemContract.java | 35 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- .../fs/s3a/ITestS3AFileSystemContract.java | 44 +- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 - .../apache/hadoop/yarn/client/cli/LogsCLI.java | 172 ++- .../hadoop/yarn/client/cli/TestLogsCLI.java | 31 + .../yarn/client/api/impl/TimelineConnector.java | 4 +- .../yarn/logaggregation/LogCLIHelpers.java | 11 +- .../client/impl/zk/RegistrySecurity.java | 12 +- .../webapp/AHSWebServices.java | 32 +- .../webapp/TestAHSWebServices.java | 17 + .../server/webapp/YarnWebServiceParams.java | 1 + .../containermanager/ContainerManagerImpl.java | 57 +- .../application/ApplicationImpl.java | 4 +- .../containermanager/container/Container.java | 2 + .../container/ContainerImpl.java | 8 + .../nodemanager/webapp/NMWebServices.java | 6 +- .../TestContainerManagerRecovery.java | 18 +- .../TestLocalCacheDirectoryManager.java | 6 +- .../nodemanager/webapp/MockContainer.java | 5 + .../nodemanager/webapp/TestNMWebServices.java | 6 + .../scheduler/capacity/CapacityScheduler.java | 2 +- .../capacity/CapacitySchedulerQueueManager.java | 7 +- .../scheduler/capacity/ParentQueue.java | 6 +- .../scheduler/policy/OrderingPolicy.java | 105 +- .../scheduler/fair/TestFairScheduler.java | 3 +- .../src/site/markdown/FairScheduler.md | 40 +- .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 13 +- .../app/components/base-chart-component.js | 4 + .../main/webapp/app/components/nodes-heatmap.js | 106 +- .../src/main/webapp/app/styles/app.css | 12 + .../hadoop-yarn-ui/src/main/webapp/package.json | 6 +- 125 files changed, 4517 insertions(+), 1071 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcc8f65a/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcc8f65a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java ---------------------------------------------------------------------- diff --cc hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java index 0274765,040e9c8..17d2404 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java @@@ -669,27 -705,13 +705,27 @@@ public abstract class FileSystemContrac /** * This a sanity check to make sure that any filesystem's handling of - * renames doesn't cause any regressions + * renames empty dirs doesn't cause any regressions. + */ + public void testRenameEmptyToDirWithSamePrefixAllowed() throws Throwable { + if (!renameSupported()) return; - Path parentdir = path("test/parentdir"); ++ Path parentdir = path("testRenameEmptyToDirWithSamePrefixAllowed"); + fs.mkdirs(parentdir); - Path dest = path("test/parentdirdest"); ++ Path dest = path("testRenameEmptyToDirWithSamePrefixAllowedDest"); + rename(parentdir, dest, true, false, true); + } + + /** + * This a sanity check to make sure that any filesystem's handling of + * renames non-empty dirs doesn't cause any regressions. */ public void testRenameToDirWithSamePrefixAllowed() throws Throwable { if (!renameSupported()) return; - Path parentdir = path("test/parentdir"); + final Path parentdir = path("testRenameToDirWithSamePrefixAllowed"); fs.mkdirs(parentdir); + // Before renaming, we create one file under the source parent directory + createFile(new Path(parentdir, "mychild")); - Path dest = path("test/parentdirdest"); + final Path dest = path("testRenameToDirWithSamePrefixAllowedDest"); rename(parentdir, dest, true, false, true); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcc8f65a/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcc8f65a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcc8f65a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
