HDFS-11368. Erasure Coding: Deprecate replication-related config keys. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de489493 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de489493 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de489493 Branch: refs/heads/HADOOP-13345 Commit: de4894936a5b581572f35fa5b8979d9f23da0891 Parents: dd98a80 Author: Andrew Wang <w...@apache.org> Authored: Wed Nov 23 16:42:06 2016 -0800 Committer: Andrew Wang <w...@apache.org> Committed: Wed Nov 23 16:42:06 2016 -0800 ---------------------------------------------------------------------- .../src/site/markdown/DeprecatedProperties.md | 7 ++- .../apache/hadoop/hdfs/HdfsConfiguration.java | 10 +++- .../hdfs/client/HdfsClientConfigKeys.java | 10 ++-- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 19 +++--- .../hadoop/hdfs/server/balancer/Balancer.java | 4 +- .../server/blockmanagement/BlockManager.java | 62 ++++++++++++-------- .../BlockPlacementPolicyDefault.java | 8 +-- .../apache/hadoop/hdfs/server/mover/Mover.java | 4 +- .../hadoop/hdfs/server/namenode/BackupNode.java | 4 +- .../src/main/resources/hdfs-default.xml | 8 +-- .../apache/hadoop/hdfs/AdminStatesBaseTest.java | 4 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 2 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +- .../TestDFSStripedOutputStreamWithFailure.java | 2 +- .../hdfs/TestDecommissionWithStriped.java | 10 ++-- .../apache/hadoop/hdfs/TestDeprecatedKeys.java | 22 ++++++- .../hadoop/hdfs/TestEncryptedTransfer.java | 2 +- .../org/apache/hadoop/hdfs/TestFileAppend4.java | 2 +- .../apache/hadoop/hdfs/TestFileChecksum.java | 2 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +- .../hadoop/hdfs/TestMissingBlocksAlert.java | 3 +- .../hdfs/TestReadStripedFileWithDecoding.java | 3 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 4 +- .../hdfs/TestReplaceDatanodeOnFailure.java | 3 +- .../org/apache/hadoop/hdfs/TestReplication.java | 3 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 2 +- .../hdfs/server/balancer/TestBalancer.java | 26 ++++---- .../blockmanagement/BlockManagerTestUtil.java | 21 ++++--- .../TestBlocksWithNotEnoughRacks.java | 4 +- .../server/blockmanagement/TestNodeCount.java | 2 +- .../TestPendingInvalidateBlock.java | 2 +- .../TestPendingReconstruction.java | 2 +- ...constructStripedBlocksWithRackAwareness.java | 8 +-- .../blockmanagement/TestReplicationPolicy.java | 2 +- .../TestReplicationPolicyConsiderLoad.java | 2 +- .../TestDataNodeErasureCodingMetrics.java | 2 +- .../fsdataset/impl/TestLazyPersistFiles.java | 12 ++-- .../hadoop/hdfs/server/mover/TestMover.java | 9 ++- .../hdfs/server/mover/TestStorageMover.java | 4 +- .../server/namenode/NNThroughputBenchmark.java | 10 ++-- .../TestAddOverReplicatedStripedBlocks.java | 2 +- .../namenode/TestDecommissioningStatus.java | 5 +- .../server/namenode/TestFSEditLogLoader.java | 2 +- .../hdfs/server/namenode/TestHostsFiles.java | 4 +- .../hdfs/server/namenode/TestMetaSave.java | 5 +- .../namenode/TestReconstructStripedBlocks.java | 9 +-- .../server/namenode/TestStripedINodeFile.java | 5 +- .../hdfs/server/namenode/ha/TestDNFencing.java | 11 ++-- .../ha/TestDNFencingWithReplication.java | 2 +- .../namenode/ha/TestPipelinesFailover.java | 5 +- .../TestNNMetricFilesInGetListingOps.java | 2 +- .../namenode/metrics/TestNameNodeMetrics.java | 12 ++-- 52 files changed, 216 insertions(+), 158 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md index 8de1be2..38a1954 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md @@ -43,8 +43,11 @@ The following table lists the configuration property names that are deprecated i | dfs.permissions | dfs.permissions.enabled | | dfs.permissions.supergroup | dfs.permissions.superusergroup | | dfs.read.prefetch.size | dfs.client.read.prefetch.size | -| dfs.replication.considerLoad | dfs.namenode.replication.considerLoad | -| dfs.replication.interval | dfs.namenode.replication.interval | +| dfs.replication.considerLoad | dfs.namenode.redundancy.considerLoad | +| dfs.namenode.replication.considerLoad | dfs.namenode.redundancy.considerLoad | +| dfs.namenode.replication.considerLoad.factor | dfs.namenode.redundancy.considerLoad.factor | +| dfs.replication.interval | dfs.namenode.redundancy.interval | +| dfs.namenode.replication.interval | dfs.namenode.redundancy.interval | | dfs.replication.min | dfs.namenode.replication.min | | dfs.replication.pending.timeout.sec | dfs.namenode.reconstruction.pending.timeout-sec | | dfs.namenode.replication.pending.timeout-sec | dfs.namenode.reconstruction.pending.timeout-sec | http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java index 3f6eae0..4d4ade2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java @@ -115,9 +115,15 @@ public class HdfsConfiguration extends Configuration { new DeprecationDelta("dfs.access.time.precision", DeprecatedKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY), new DeprecationDelta("dfs.replication.considerLoad", - DeprecatedKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY), + DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY), + new DeprecationDelta("dfs.namenode.replication.considerLoad", + DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY), + new DeprecationDelta("dfs.namenode.replication.considerLoad.factor", + DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR), new DeprecationDelta("dfs.replication.interval", - DeprecatedKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY), + DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY), + new DeprecationDelta("dfs.namenode.replication.interval", + DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY), new DeprecationDelta("dfs.replication.min", DeprecatedKeys.DFS_NAMENODE_REPLICATION_MIN_KEY), new DeprecationDelta("dfs.replication.pending.timeout.sec", http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 4c754d9..7ad79e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -204,10 +204,12 @@ public interface HdfsClientConfigKeys { String DFS_METRICS_SESSION_ID_KEY = "dfs.metrics.session-id"; String DFS_NAMENODE_ACCESSTIME_PRECISION_KEY = "dfs.namenode.accesstime.precision"; - String DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY = - "dfs.namenode.replication.considerLoad"; - String DFS_NAMENODE_REPLICATION_INTERVAL_KEY = - "dfs.namenode.replication.interval"; + String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY = + "dfs.namenode.redundancy.considerLoad"; + String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR = + "dfs.namenode.redundancy.considerLoad.factor"; + String DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY = + "dfs.namenode.redundancy.interval.seconds"; String DFS_NAMENODE_REPLICATION_MIN_KEY = "dfs.namenode.replication.min"; String DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY = "dfs.namenode.reconstruction.pending.timeout-sec"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index b9fd939..d7d3c9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -194,16 +194,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_ACCESSTIME_PRECISION_KEY = HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY; public static final long DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT = 3600000; - public static final String DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY = - HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY; - public static final boolean DFS_NAMENODE_REPLICATION_CONSIDERLOAD_DEFAULT = true; - public static final String DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR = - "dfs.namenode.replication.considerLoad.factor"; + public static final String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY = + HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY; + public static final boolean DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_DEFAULT = + true; + public static final String DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR = + "dfs.namenode.redundancy.considerLoad.factor"; public static final double - DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR_DEFAULT = 2.0; - public static final String DFS_NAMENODE_REPLICATION_INTERVAL_KEY = - HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY; - public static final int DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT = 3; + DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR_DEFAULT = 2.0; + public static final String DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY = + HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY; + public static final int DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT = 3; public static final String DFS_NAMENODE_REPLICATION_MIN_KEY = HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_REPLICATION_MIN_KEY; public static final int DFS_NAMENODE_REPLICATION_MIN_DEFAULT = 1; http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 61352f3..75eecde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -673,8 +673,8 @@ public class Balancer { DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 2000 + conf.getTimeDuration( - DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT, + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT, TimeUnit.SECONDS) * 1000; LOG.info("namenodes = " + namenodes); LOG.info("parameters = " + p); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 2f57571..1b744e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -230,8 +230,11 @@ public class BlockManager implements BlockStatsMXBean { return pendingReconstruction.getNumTimedOuts(); } - /**replicationRecheckInterval is how often namenode checks for new replication work*/ - private final long replicationRecheckInterval; + /** + * redundancyRecheckInterval is how often namenode checks for new + * reconstruction work. + */ + private final long redundancyRecheckIntervalMs; /** How often to check and the limit for the storageinfo efficiency. */ private final long storageInfoDefragmentInterval; @@ -244,8 +247,8 @@ public class BlockManager implements BlockStatsMXBean { */ final BlocksMap blocksMap; - /** Replication thread. */ - final Daemon replicationThread = new Daemon(new ReplicationMonitor()); + /** Redundancy thread. */ + private final Daemon redundancyThread = new Daemon(new RedundancyMonitor()); /** StorageInfoDefragmenter thread. */ private final Daemon storageInfoDefragmenterThread = @@ -435,10 +438,10 @@ public class BlockManager implements BlockStatsMXBean { this.blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf); this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf); - this.replicationRecheckInterval = - conf.getTimeDuration(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT, - TimeUnit.SECONDS) * 1000L; + this.redundancyRecheckIntervalMs = conf.getTimeDuration( + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT, + TimeUnit.SECONDS) * 1000; this.storageInfoDefragmentInterval = conf.getLong( @@ -493,7 +496,8 @@ public class BlockManager implements BlockStatsMXBean { LOG.info("maxReplication = " + maxReplication); LOG.info("minReplication = " + minReplication); LOG.info("maxReplicationStreams = " + maxReplicationStreams); - LOG.info("replicationRecheckInterval = " + replicationRecheckInterval); + LOG.info("redundancyRecheckInterval = " + redundancyRecheckIntervalMs + + "ms"); LOG.info("encryptDataTransfer = " + encryptDataTransfer); LOG.info("maxNumBlocksToLog = " + maxNumBlocksToLog); } @@ -586,7 +590,7 @@ public class BlockManager implements BlockStatsMXBean { return blockTokenSecretManager; } - /** Allow silent termination of replication monitor for testing */ + /** Allow silent termination of redundancy monitor for testing. */ @VisibleForTesting void enableRMTerminationForTesting() { checkNSRunning = false; @@ -604,8 +608,8 @@ public class BlockManager implements BlockStatsMXBean { public void activate(Configuration conf, long blockTotal) { pendingReconstruction.start(); datanodeManager.activate(conf); - this.replicationThread.setName("ReplicationMonitor"); - this.replicationThread.start(); + this.redundancyThread.setName("RedundancyMonitor"); + this.redundancyThread.start(); storageInfoDefragmenterThread.setName("StorageInfoMonitor"); storageInfoDefragmenterThread.start(); this.blockReportThread.start(); @@ -616,10 +620,10 @@ public class BlockManager implements BlockStatsMXBean { public void close() { bmSafeMode.close(); try { - replicationThread.interrupt(); + redundancyThread.interrupt(); storageInfoDefragmenterThread.interrupt(); blockReportThread.interrupt(); - replicationThread.join(3000); + redundancyThread.join(3000); storageInfoDefragmenterThread.join(3000); blockReportThread.join(3000); } catch (InterruptedException ie) { @@ -880,7 +884,7 @@ public class BlockManager implements BlockStatsMXBean { /** * If IBR is not sent from expected locations yet, add the datanodes to - * pendingReconstruction in order to keep ReplicationMonitor from scheduling + * pendingReconstruction in order to keep RedundancyMonitor from scheduling * the block. */ public void addExpectedReplicasToPending(BlockInfo blk) { @@ -1884,7 +1888,7 @@ public class BlockManager implements BlockStatsMXBean { if (hasEnoughEffectiveReplicas(block, numReplicas, pendingNum)) { neededReconstruction.remove(block, priority); rw.resetTargets(); - blockLog.debug("BLOCK* Removing {} from neededReplications as" + + blockLog.debug("BLOCK* Removing {} from neededReconstruction as" + " it has enough replicas", block); return false; } @@ -1910,8 +1914,8 @@ public class BlockManager implements BlockStatsMXBean { // reconstructions that fail after an appropriate amount of time. pendingReconstruction.increment(block, DatanodeStorageInfo.toDatanodeDescriptors(targets)); - blockLog.debug("BLOCK* block {} is moved from neededReplications to " - + "pendingReplications", block); + blockLog.debug("BLOCK* block {} is moved from neededReconstruction to " + + "pendingReconstruction", block); int numEffectiveReplicas = numReplicas.liveReplicas() + pendingNum; // remove from neededReconstruction @@ -4298,32 +4302,32 @@ public class BlockManager implements BlockStatsMXBean { /** * Periodically calls computeBlockRecoveryWork(). */ - private class ReplicationMonitor implements Runnable { + private class RedundancyMonitor implements Runnable { @Override public void run() { while (namesystem.isRunning()) { try { - // Process replication work only when active NN is out of safe mode. + // Process recovery work only when active NN is out of safe mode. if (isPopulatingReplQueues()) { computeDatanodeWork(); processPendingReconstructions(); rescanPostponedMisreplicatedBlocks(); } - Thread.sleep(replicationRecheckInterval); + TimeUnit.MILLISECONDS.sleep(redundancyRecheckIntervalMs); } catch (Throwable t) { if (!namesystem.isRunning()) { - LOG.info("Stopping ReplicationMonitor."); + LOG.info("Stopping RedundancyMonitor."); if (!(t instanceof InterruptedException)) { - LOG.info("ReplicationMonitor received an exception" + LOG.info("RedundancyMonitor received an exception" + " while shutting down.", t); } break; } else if (!checkNSRunning && t instanceof InterruptedException) { - LOG.info("Stopping ReplicationMonitor for testing."); + LOG.info("Stopping RedundancyMonitor for testing."); break; } - LOG.error("ReplicationMonitor thread received Runtime exception. ", + LOG.error("RedundancyMonitor thread received Runtime exception. ", t); terminate(1, t); } @@ -4692,6 +4696,14 @@ public class BlockManager implements BlockStatsMXBean { } } + /** + * @return redundancy thread. + */ + @VisibleForTesting + Daemon getRedundancyThread() { + return redundancyThread; + } + public BlockIdManager getBlockIdManager() { return blockIdManager; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index cb65ba8..eb54667 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -83,11 +83,11 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { NetworkTopology clusterMap, Host2NodesMap host2datanodeMap) { this.considerLoad = conf.getBoolean( - DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_DEFAULT); + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_DEFAULT); this.considerLoadFactor = conf.getDouble( - DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR_DEFAULT); + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR, + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR_DEFAULT); this.stats = stats; this.clusterMap = clusterMap; this.host2datanodeMap = host2datanodeMap; http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index 4fbae76..4ab55d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -610,8 +610,8 @@ public class Mover { DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 2000 + conf.getTimeDuration( - DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT, + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT, TimeUnit.SECONDS) * 1000; AtomicInteger retryCount = new AtomicInteger(0); LOG.info("namenodes = " + namenodes); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java index 899f0d7..b859148 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java @@ -469,11 +469,11 @@ public class BackupNode extends NameNode { * (not run or not pass any control commands to DataNodes) * on BackupNode: * {@link LeaseManager.Monitor} protected by SafeMode. - * {@link BlockManager.ReplicationMonitor} protected by SafeMode. + * {@link BlockManager.RedundancyMonitor} protected by SafeMode. * {@link HeartbeatManager.Monitor} protected by SafeMode. * {@link DecommissionManager.Monitor} need to prohibit refreshNodes(). * {@link PendingReconstructionBlocks.PendingReconstructionMonitor} - * harmless, because ReplicationMonitor is muted. + * harmless, because RedundancyMonitor is muted. */ @Override public void startActiveServices() throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index acb24fc..c9d74bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -296,14 +296,14 @@ </property> <property> - <name>dfs.namenode.replication.considerLoad</name> + <name>dfs.namenode.redundancy.considerLoad</name> <value>true</value> <description>Decide if chooseTarget considers the target's load or not </description> </property> <property> - <name>dfs.namenode.replication.considerLoad.factor</name> + <name>dfs.namenode.redundancy.considerLoad.factor</name> <value>2.0</value> <description>The factor by which a node's load can exceed the average before being rejected for writes, only if considerLoad is true. @@ -980,10 +980,10 @@ </property> <property> - <name>dfs.namenode.replication.interval</name> + <name>dfs.namenode.redundancy.interval.seconds</name> <value>3s</value> <description>The periodicity in seconds with which the namenode computes - replication work for datanodes. Support multiple time unit suffix(case insensitive), + low redundancy work for datanodes. Support multiple time unit suffix(case insensitive), as described in dfs.heartbeat.interval. </description> </property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java index 534c5e0..0ed01f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java @@ -90,14 +90,14 @@ public class AdminStatesBaseTest { } // Setup conf - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 200); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL); conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, BLOCKREPORT_INTERVAL_MSEC); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, NAMENODE_REPLICATION_INTERVAL); conf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index 624fc74..6d55c46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -58,7 +58,7 @@ public class TestBlockStoragePolicy { static { conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); POLICY_SUITE = BlockStoragePolicySuite.createDefaultSuite(); DEFAULT_STORAGE_POLICY = POLICY_SUITE.getDefaultPolicy(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 47d5e02..b686f28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -67,7 +67,7 @@ public class TestDFSStripedOutputStream { int numDNs = dataBlocks + parityBlocks + 2; conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index 53059ce..cde07a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure { private HdfsConfiguration newHdfsConfiguration() { final HdfsConfiguration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java index 1574a03..2eab309 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java @@ -111,26 +111,26 @@ public class TestDecommissionWithStriped { writeConfigFile(excludeFile, null); // Setup conf - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); conf.set(DFSConfigKeys.DFS_HOSTS, hostsFile.toUri().getPath()); conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath()); conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 2000); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, BLOCKREPORT_INTERVAL_MSEC); conf.setInt(DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, 4); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, NAMENODE_REPLICATION_INTERVAL); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt( DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY, cellSize - 1); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); numDNs = dataBlocks + parityBlocks + 2; http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java index 7152092..ad5c2a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeprecatedKeys.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import org.apache.hadoop.conf.Configuration; import org.junit.Test; @@ -33,8 +34,23 @@ public class TestDeprecatedKeys { String scriptFile = conf.get(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY); assertTrue(scriptFile.equals("xyz")) ; conf.setInt("dfs.replication.interval", 1); - String alpha = DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY; - int repInterval = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 3) ; - assertTrue(repInterval == 1) ; + int redundancyInterval = conf + .getInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 3); + assertTrue(redundancyInterval == 1); + int repInterval = conf.getInt("dfs.replication.interval", 3); + assertTrue(repInterval == 1); + repInterval = conf.getInt("dfs.namenode.replication.interval", 3); + assertTrue(repInterval == 1); + + conf.setBoolean("dfs.replication.considerLoad", false); + assertFalse(conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, true)); + assertFalse(conf.getBoolean("dfs.replication.considerLoad", true)); + + conf.setDouble("dfs.namenode.replication.considerLoad.factor", 5.0); + assertTrue(5.0 == conf.getDouble( + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR, 2.0)); + assertTrue(5.0 == conf + .getDouble("dfs.namenode.replication.considerLoad.factor", 2.0)); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java index 78020b2..cc90863 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptedTransfer.java @@ -329,7 +329,7 @@ public class TestEncryptedTransfer { // client only retries establishing pipeline with the 4th node. int numDataNodes = 4; // do not consider load factor when selecting a data node - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); setEncryptionConfigKeys(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java index ae0f0c2..40bc314 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend4.java @@ -86,7 +86,7 @@ public class TestFileAppend4 { // handle under-replicated blocks quickly (for replication asserts) conf.setInt( DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, 5); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); // handle failures in the DFSClient pipeline quickly // (for cluster.shutdown(); fs.close() idiom) http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java index 1c75f90..9d60798 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java @@ -75,7 +75,7 @@ public class TestFileChecksum { int numDNs = dataBlocks + parityBlocks + 2; conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java index ca8f6db..246818b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java @@ -85,7 +85,7 @@ public class TestLeaseRecoveryStriped { conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setLong(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 6000L); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java index 90d42b6..e225141 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java @@ -59,7 +59,8 @@ public class TestMissingBlocksAlert { try { Configuration conf = new HdfsConfiguration(); //minimize test delay - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 0); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 0); conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10); int fileLen = 10*1024; conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, fileLen/2); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java index 6014332..506a005 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java @@ -99,7 +99,8 @@ public class TestReadStripedFileWithDecoding { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY, 0); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, + false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); fs = cluster.getFileSystem(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index a3d9ef6..4960c58 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -95,8 +95,8 @@ public class TestReconstructStripedFile { conf.setInt( DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY, cellSize - 1); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java index 3dda7e8..aa5c70f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java @@ -118,7 +118,8 @@ public class TestReplaceDatanodeOnFailure { public void testReplaceDatanodeOnFailure() throws Exception { final Configuration conf = new HdfsConfiguration(); // do not consider load factor when selecting a data node - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, + false); //always replace a datanode ReplaceDatanodeOnFailure.write(Policy.ALWAYS, true, conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java index 3e23db3..0f1bedd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java @@ -228,7 +228,8 @@ public class TestReplication { */ public void runReplication(boolean simulated) throws IOException { Configuration conf = new HdfsConfiguration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, + false); if (simulated) { SimulatedFSDataset.setFactory(conf); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index 8ada593..712f326 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -75,7 +75,7 @@ public class TestWriteReadStripedFile { @Before public void setup() throws IOException { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); fs = cluster.getFileSystem(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 730f655..1906d74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -179,7 +179,8 @@ public class TestBalancer { conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1L); SimulatedFSDataset.setFactory(conf); conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); @@ -212,10 +213,12 @@ public class TestBalancer { void initConfWithStripe(Configuration conf) { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultBlockSize); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, + false); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); SimulatedFSDataset.setFactory(conf); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1L); conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L); } @@ -922,11 +925,13 @@ public class TestBalancer { private static int runBalancer(Collection<URI> namenodes, final BalancerParameters p, Configuration conf) throws IOException, InterruptedException { - final long sleeptime = - conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, - DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 + - conf.getLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000; + final long sleeptime = conf.getLong( + DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, + DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 + + conf.getLong( + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT) + * 1000; LOG.info("namenodes = " + namenodes); LOG.info("parameters = " + p); LOG.info("Print stack trace", new Throwable()); @@ -1603,7 +1608,7 @@ public class TestBalancer { Configuration conf = new HdfsConfiguration(); conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L); @@ -1670,7 +1675,8 @@ public class TestBalancer { int blockSize = 5 * 1024 * 1024 ; conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1L); conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 5fb3b25..1c7442a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -105,25 +105,24 @@ public class BlockManagerTestUtil { } /** - * @return replication monitor thread instance from block manager. + * @return redundancy monitor thread instance from block manager. */ - public static Daemon getReplicationThread(final BlockManager blockManager) - { - return blockManager.replicationThread; + public static Daemon getRedundancyThread(final BlockManager blockManager) { + return blockManager.getRedundancyThread(); } - + /** - * Stop the replication monitor thread + * Stop the redundancy monitor thread. */ - public static void stopReplicationThread(final BlockManager blockManager) + public static void stopRedundancyThread(final BlockManager blockManager) throws IOException { blockManager.enableRMTerminationForTesting(); - blockManager.replicationThread.interrupt(); + blockManager.getRedundancyThread().interrupt(); try { - blockManager.replicationThread.join(); - } catch(InterruptedException ie) { + blockManager.getRedundancyThread().join(); + } catch (InterruptedException ie) { throw new IOException( - "Interrupted while trying to stop ReplicationMonitor"); + "Interrupted while trying to stop RedundancyMonitor"); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java index 197e3a6..0298cbc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlocksWithNotEnoughRacks.java @@ -62,9 +62,9 @@ public class TestBlocksWithNotEnoughRacks { // commands quickly (as replies to heartbeats) conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - // Have the NN ReplicationMonitor compute the replication and + // Have the NN RedundancyMonitor compute the reconstruction and // invalidation commands to send DNs every second. - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); // Have the NN check for pending replications every second so it // quickly schedules additional replicas as they are identified. http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java index 6058727..d915b6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java @@ -56,7 +56,7 @@ public class TestNodeCount { 60); // reduce intervals to make test execution time shorter - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); // start a mini dfs cluster of 2 nodes http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java index d856065..674f883 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java @@ -65,7 +65,7 @@ public class TestPendingInvalidateBlock { // set the block report interval to 2s conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 2000); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); // disable the RPC timeout for debug conf.setLong(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, 0); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java index ba3be91..aeaaf4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReconstruction.java @@ -374,7 +374,7 @@ public class TestPendingReconstruction { CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024); CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, DFS_REPLICATION_INTERVAL); - CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, + CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, DFS_REPLICATION_INTERVAL); MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes( DATANODE_COUNT).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java index 94ee623..ace8732 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java @@ -96,8 +96,8 @@ public class TestReconstructStripedBlocksWithRackAwareness { @BeforeClass public static void setup() throws Exception { - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); conf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 1); } @@ -139,7 +139,7 @@ public class TestReconstructStripedBlocksWithRackAwareness { * * In this test, we first need to create a scenario that a striped block has * all the internal blocks but distributed in <6 racks. Then we check if the - * replication monitor can correctly schedule the reconstruction work for it. + * redundancy monitor can correctly schedule the reconstruction work for it. */ @Test public void testReconstructForNotEnoughRacks() throws Exception { @@ -194,7 +194,7 @@ public class TestReconstructStripedBlocksWithRackAwareness { fsn.writeUnlock(); } - // check if replication monitor correctly schedule the replication work + // check if redundancy monitor correctly schedule the reconstruction work. boolean scheduled = false; for (int i = 0; i < 5; i++) { // retry 5 times for (DatanodeStorageInfo storage : blockInfo.storages) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 9c62383..1af013d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -836,7 +836,7 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest { int DFS_NAMENODE_REPLICATION_INTERVAL = 1000; int HIGH_PRIORITY = 0; Configuration conf = new Configuration(); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2) .format(true).build(); try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java index 8336fec..bcd8245 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java @@ -53,7 +53,7 @@ public class TestReplicationPolicyConsiderLoad @Override DatanodeDescriptor[] getDatanodeDescriptors(Configuration conf) { conf.setDouble(DFSConfigKeys - .DFS_NAMENODE_REPLICATION_CONSIDERLOAD_FACTOR, 1.2); + .DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_FACTOR, 1.2); final String[] racks = { "/rack1", "/rack1", http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java index 98efc13c..1b0526b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java @@ -78,7 +78,7 @@ public class TestDataNodeErasureCodingMetrics { conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java index 8c43592..d268d01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java @@ -104,8 +104,8 @@ public class TestLazyPersistFiles extends LazyPersistTestCase { Thread.sleep(30000L); assertThat(cluster.getNamesystem().getNumDeadDataNodes(), is(1)); - // Next, wait for the replication monitor to mark the file as corrupt - Thread.sleep(2 * DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT * 1000); + // Next, wait for the redundancy monitor to mark the file as corrupt. + Thread.sleep(2 * DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT * 1000); // Wait for the LazyPersistFileScrubber to run Thread.sleep(2 * LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC * 1000); @@ -137,8 +137,8 @@ public class TestLazyPersistFiles extends LazyPersistTestCase { cluster.shutdownDataNodes(); Thread.sleep(30000L); - // Next, wait for the replication monitor to mark the file as corrupt - Thread.sleep(2 * DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT * 1000); + // Next, wait for the redundancy monitor to mark the file as corrupt. + Thread.sleep(2 * DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT * 1000); // Wait for the LazyPersistFileScrubber to run Thread.sleep(2 * LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC * 1000); @@ -164,8 +164,8 @@ public class TestLazyPersistFiles extends LazyPersistTestCase { cluster.restartNameNodes(); - // wait for the replication monitor to mark the file as corrupt - Thread.sleep(2 * DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT * 1000); + // wait for the redundancy monitor to mark the file as corrupt. + Thread.sleep(2 * DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_DEFAULT * 1000); Long corruptBlkCount = (long) Iterators.size(cluster.getNameNode() .getNamesystem().getBlockManager().getCorruptReplicaBlockIterator()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java index 4342dab..20a6959 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java @@ -105,7 +105,8 @@ public class TestMover { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE); conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1L); conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L); } @@ -482,8 +483,10 @@ public class TestMover { void initConfWithStripe(Configuration conf) { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultBlockSize); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1L); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, + false); } @Test(timeout = 300000) http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java index 1b5bd81..077997f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java @@ -93,8 +93,8 @@ public class TestStorageMover { static { DEFAULT_CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); DEFAULT_CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - DEFAULT_CONF.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, - 2L); + DEFAULT_CONF.setLong( + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 2L); DEFAULT_CONF.setLong(DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_KEY, 2000L); DEFAULT_POLICIES = BlockStoragePolicySuite.createDefaultSuite(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index 4b9db07..a3d0be5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -1237,8 +1237,8 @@ public class NNThroughputBenchmark implements Tool { } // end BlockReportStats /** - * Measures how fast replication monitor can compute data-node work. - * + * Measures how fast redundancy monitor can compute data-node work. + * * It runs only one thread until no more work can be scheduled. */ class ReplicationStats extends OperationStatsBase { @@ -1265,7 +1265,7 @@ public class NNThroughputBenchmark implements Tool { parseArguments(args); // number of operations is 4 times the number of decommissioned // blocks divided by the number of needed replications scanned - // by the replication monitor in one iteration + // by the redundancy monitor in one iteration numOpsRequired = (totalBlocks*replication*nodesToDecommission*2) / (numDatanodes*numDatanodes); @@ -1314,8 +1314,8 @@ public class NNThroughputBenchmark implements Tool { // start data-nodes; create a bunch of files; generate block reports. blockReportObject.generateInputs(ignore); - // stop replication monitor - BlockManagerTestUtil.stopReplicationThread(namesystem.getBlockManager()); + // stop redundancy monitor thread. + BlockManagerTestUtil.stopRedundancyThread(namesystem.getBlockManager()); // report blocks once int nrDatanodes = blockReportObject.getNumDatanodes(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java index 15a8756..c8b25564 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java @@ -74,7 +74,7 @@ public class TestAddOverReplicatedStripedBlocks { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); // disable block recovery conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); SimulatedFSDataset.setFactory(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java index 7f1cc9a..8bdaa74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java @@ -78,7 +78,7 @@ public class TestDecommissioningStatus { @Before public void setUp() throws Exception { conf = new HdfsConfiguration(); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); // Set up the hosts/exclude files. @@ -89,7 +89,8 @@ public class TestDecommissioningStatus { conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt( DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY, 4); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1000); conf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 1); conf.setLong(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY, 1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index f6e99ee..a57a936 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -156,7 +156,7 @@ public class TestFSEditLogLoader { // start a cluster Configuration conf = getConf(); // Replicate and heartbeat fast to shave a few seconds off test - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); MiniDFSCluster cluster = null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java index cb19c2c..79e7acc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java @@ -76,9 +76,9 @@ public class TestHostsFiles { // commands quickly (as replies to heartbeats) conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - // Have the NN ReplicationMonitor compute the replication and + // Have the NN RedundancyMonitor compute the low redundant blocks and // invalidation commands to send DNs every second. - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); // Have the NN check for pending replications every second so it // quickly schedules additional replicas as they are identified. http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java index 9816d00..0303a5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java @@ -64,8 +64,9 @@ public class TestMetaSave { Configuration conf = new HdfsConfiguration(); // High value of replication interval - // so that blocks remain under-replicated - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000); + // so that blocks remain less redundant + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1000); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); conf.setLong(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1L); conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, 5L); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java index d43804a..cf1585c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java @@ -192,7 +192,8 @@ public class TestReconstructStripedBlocks { public void test2RecoveryTasksForSameBlockGroup() throws Exception { Configuration conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1000); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1000); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2) .build(); @@ -255,8 +256,8 @@ public class TestReconstructStripedBlocks { @Test public void testCountLiveReplicas() throws Exception { final HdfsConfiguration conf = new HdfsConfiguration(); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2) .build(); @@ -299,7 +300,7 @@ public class TestReconstructStripedBlocks { FSNamesystem fsn = cluster.getNamesystem(); BlockManager bm = fsn.getBlockManager(); - Thread.sleep(3000); // wait 3 running cycles of replication monitor + Thread.sleep(3000); // wait 3 running cycles of redundancy monitor for (DataNode dn : cluster.getDataNodes()) { DataNodeTestUtils.triggerHeartbeat(dn); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index 6c4d73c..d92ec1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -307,8 +307,9 @@ public class TestStripedINodeFile { int defaultStripedBlockSize = testECPolicy.getCellSize() * 4; conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultStripedBlockSize); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L); - conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, + conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 1L); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); // start 10 datanodes http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java index 15357b0..c1521bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java @@ -87,9 +87,10 @@ public class TestDNFencing { public void setupCluster() throws Exception { conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, SMALL_BLOCK); - // Bump up replication interval so that we only run replication + // Bump up redundancy interval so that we only run low redundancy // checks explicitly. - conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 600); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, + 600); // Increase max streams so that we re-replicate quickly. conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 1000); // See RandomDeleterPolicy javadoc. @@ -167,7 +168,7 @@ public class TestDNFencing { // The blocks should no longer be postponed. assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks()); - // Wait for NN2 to enact its deletions (replication monitor has to run, etc) + // Wait for NN2 to enact its deletions (redundancy monitor has to run, etc) BlockManagerTestUtil.computeInvalidationWork( nn2.getNamesystem().getBlockManager()); cluster.triggerHeartbeats(); @@ -258,7 +259,7 @@ public class TestDNFencing { // The block should no longer be postponed. assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks()); - // Wait for NN2 to enact its deletions (replication monitor has to run, etc) + // Wait for NN2 to enact its deletions (redundancy monitor has to run, etc) BlockManagerTestUtil.computeInvalidationWork( nn2.getNamesystem().getBlockManager()); @@ -358,7 +359,7 @@ public class TestDNFencing { // The block should no longer be postponed. assertEquals(0, nn2.getNamesystem().getPostponedMisreplicatedBlocks()); - // Wait for NN2 to enact its deletions (replication monitor has to run, etc) + // Wait for NN2 to enact its deletions (redundancy monitor has to run, etc) BlockManagerTestUtil.computeInvalidationWork( nn2.getNamesystem().getBlockManager()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/de489493/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java index e4fe230..c91d4de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencingWithReplication.java @@ -110,7 +110,7 @@ public class TestDNFencingWithReplication { harness.conf.setInt( DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1); harness.conf.setInt( - DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1); + DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); final MiniDFSCluster cluster = harness.startCluster(); try { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
