Merge remote-tracking branch 'apache-commit/trunk' into ozone
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/845a7106 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/845a7106 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/845a7106 Branch: refs/heads/HDFS-7240 Commit: 845a71063178bff240e14c6943bdcec94649e6dc Parents: f08bf36 dd4b387 Author: Arpit Agarwal <a...@apache.org> Authored: Wed Jun 24 10:45:56 2015 -0700 Committer: Arpit Agarwal <a...@apache.org> Committed: Wed Jun 24 10:45:56 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/ha/ZKFailoverController.java | 61 ++- .../java/org/apache/hadoop/util/LineReader.java | 8 + .../org/apache/hadoop/ha/MiniZKFCCluster.java | 93 +++- .../hadoop/ha/TestZKFailoverController.java | 32 ++ hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 10 + .../org/apache/hadoop/hdfs/DFSUtilClient.java | 7 +- .../hdfs/protocol/BlockStoragePolicy.java | 12 +- .../hadoop/hdfs/protocol/CachePoolInfo.java | 3 - .../apache/hadoop/hdfs/web/JsonUtilClient.java | 10 + .../org/apache/hadoop/hdfs/web/TokenAspect.java | 16 +- .../hadoop/hdfs/web/URLConnectionFactory.java | 13 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 60 ++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 21 +- .../bkjournal/TestBookKeeperHACheckpoints.java | 7 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 + .../java/org/apache/hadoop/hdfs/HAUtil.java | 36 +- .../token/block/BlockTokenSecretManager.java | 40 +- .../server/blockmanagement/BlockManager.java | 43 +- .../hdfs/server/namenode/CheckpointConf.java | 14 +- .../hdfs/server/namenode/ImageServlet.java | 88 +++- .../hdfs/server/namenode/NNUpgradeUtil.java | 18 +- .../server/namenode/NameNodeHttpServer.java | 7 +- .../hdfs/server/namenode/TransferFsImage.java | 47 +- .../server/namenode/ha/BootstrapStandby.java | 94 ++-- .../hdfs/server/namenode/ha/EditLogTailer.java | 162 +++++-- .../server/namenode/ha/RemoteNameNodeInfo.java | 133 ++++++ .../server/namenode/ha/StandbyCheckpointer.java | 182 +++++--- .../web/resources/NamenodeWebHdfsMethods.java | 12 +- .../hdfs/tools/DFSZKFailoverController.java | 13 + .../src/main/resources/hdfs-default.xml | 20 + .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 464 ++++++++++++------- .../apache/hadoop/hdfs/MiniDFSNNTopology.java | 18 +- .../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 58 ++- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 6 +- .../apache/hadoop/hdfs/TestRollingUpgrade.java | 109 +++-- .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 94 ++-- .../security/token/block/TestBlockToken.java | 10 +- .../hdfs/server/namenode/TestBackupNode.java | 2 +- .../hdfs/server/namenode/TestCheckpoint.java | 3 +- .../server/namenode/ha/HAStressTestHarness.java | 46 +- .../hdfs/server/namenode/ha/HATestUtil.java | 60 ++- .../namenode/ha/TestBootstrapStandby.java | 176 ++++--- .../ha/TestBootstrapStandbyWithQJM.java | 47 +- .../ha/TestDNFencingWithReplication.java | 1 + .../server/namenode/ha/TestEditLogTailer.java | 12 +- .../ha/TestFailoverWithBlockTokensEnabled.java | 55 ++- .../server/namenode/ha/TestHAConfiguration.java | 49 +- .../namenode/ha/TestPipelinesFailover.java | 110 +++-- .../namenode/ha/TestRemoteNameNodeInfo.java | 61 +++ .../namenode/ha/TestSeveralNameNodes.java | 179 +++++++ .../namenode/ha/TestStandbyCheckpoints.java | 106 +++-- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 59 +++ .../hdfs/web/TestWebHdfsFileSystemContract.java | 2 +- .../hadoop/test/MiniDFSClusterManager.java | 4 + .../src/test/resources/hadoop-0.23-reserved.tgz | Bin 4558 -> 5590 bytes .../src/test/resources/hadoop-1-reserved.tgz | Bin 2572 -> 3348 bytes .../src/test/resources/hadoop-2-reserved.tgz | Bin 2838 -> 3465 bytes .../src/test/resources/hadoop-22-dfs-dir.tgz | Bin 318180 -> 413239 bytes .../src/test/resources/hadoop1-bbw.tgz | Bin 40234 -> 43294 bytes .../src/test/resources/log4j.properties | 2 +- hadoop-mapreduce-project/CHANGES.txt | 20 + .../mapreduce/v2/app/webapp/JobBlock.java | 2 + .../mapreduce/v2/app/webapp/dao/JobInfo.java | 6 + .../v2/app/webapp/TestAMWebServicesJobs.java | 2 +- .../apache/hadoop/mapred/LineRecordReader.java | 4 +- .../mapreduce/lib/input/LineRecordReader.java | 3 +- .../lib/input/UncompressedSplitLineReader.java | 125 +++++ .../lib/output/FileOutputCommitter.java | 2 +- .../hadoop/mapreduce/task/reduce/Fetcher.java | 1 + .../task/reduce/IFileWrappedMapOutput.java | 10 +- .../mapreduce/task/reduce/LocalFetcher.java | 15 +- .../src/main/resources/mapred-default.xml | 2 +- .../hadoop/mapred/TestLineRecordReader.java | 77 ++- .../lib/input/TestLineRecordReader.java | 79 +++- .../mapreduce/v2/hs/server/HSAdminServer.java | 43 +- .../v2/hs/server/TestHSAdminServer.java | 55 +++ .../java/org/apache/hadoop/hdfs/NNBench.java | 3 +- hadoop-yarn-project/CHANGES.txt | 18 + .../hadoop/yarn/conf/YarnConfiguration.java | 10 + .../apache/hadoop/yarn/client/ServerProxy.java | 2 + .../src/main/resources/yarn-default.xml | 12 + .../localizer/ResourceLocalizationService.java | 31 +- .../containermanager/TestNMProxy.java | 20 +- .../TestResourceLocalizationService.java | 10 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 4 + .../amlauncher/ApplicationMasterLauncher.java | 30 +- .../scheduler/fair/FairScheduler.java | 11 +- 87 files changed, 2716 insertions(+), 880 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/845a7106/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 6598892,7d3a678..4b760b7 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@@ -17,14 -17,34 +17,38 @@@ */ package org.apache.hadoop.hdfs.server.blockmanagement; + import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; + import static org.apache.hadoop.util.ExitUtil.terminate; + + import java.io.IOException; + import java.io.PrintWriter; + import java.util.ArrayList; + import java.util.Collection; + import java.util.Collections; + import java.util.EnumSet; + import java.util.HashMap; + import java.util.HashSet; + import java.util.Iterator; + import java.util.LinkedList; + import java.util.List; + import java.util.Map; + import java.util.Queue; + import java.util.Set; + import java.util.TreeMap; + import java.util.TreeSet; + import java.util.concurrent.ThreadLocalRandom; + import java.util.concurrent.atomic.AtomicLong; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.StorageType; + import org.apache.hadoop.hdfs.DFSUtilClient; + import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil;
