Author: suresh
Date: Sat Aug 18 19:50:15 2012
New Revision: 1374645
URL: http://svn.apache.org/viewvc?rev=1374645&view=rev
Log:
HDFS-3816. Invalidate work percentage default value should be 0.32f.
Contributed by Jing Zhao
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1374645&r1=1374644&r2=1374645&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Sat Aug 18
19:50:15 2012
@@ -634,6 +634,9 @@ Branch-2 ( Unreleased changes )
HDFS-3788. ByteRangeInputStream should not expect HTTP Content-Length
header
when chunked transfer-encoding is used. (szetszwo)
+ HDFS-3816. Invalidate work percentage default value should be 0.32f
+ instead of 32. (Jing Zhao via suresh)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1374645&r1=1374644&r2=1374645&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Sat Aug 18 19:50:15 2012
@@ -178,7 +178,7 @@ public class DFSConfigKeys extends Commo
// Replication monitoring related keys
public static final String DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION =
"dfs.namenode.invalidate.work.pct.per.iteration";
- public static final int
DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT = 32;
+ public static final float
DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT = 0.32f;
public static final String
DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION =
"dfs.namenode.replication.work.multiplier.per.iteration";
public static final int
DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT = 2;
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1374645&r1=1374644&r2=1374645&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
Sat Aug 18 19:50:15 2012
@@ -80,6 +80,7 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.protobuf.BlockingService;
@@ -1143,4 +1144,42 @@ public class DFSUtil {
}
return false;
}
+
+ /**
+ * Get DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION from configuration.
+ *
+ * @param conf Configuration
+ * @return Value of DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION
+ */
+ public static float getInvalidateWorkPctPerIteration(Configuration conf) {
+ float blocksInvalidateWorkPct = conf.getFloat(
+ DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,
+ DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT);
+ Preconditions.checkArgument(
+ (blocksInvalidateWorkPct > 0),
+ DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION +
+ " = '" + blocksInvalidateWorkPct + "' is invalid. " +
+ "It should be a positive, non-zero float value " +
+ "indicating a percentage.");
+ return blocksInvalidateWorkPct;
+ }
+
+ /**
+ * Get DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION from
+ * configuration.
+ *
+ * @param conf Configuration
+ * @return Value of DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION
+ */
+ public static int getReplWorkMultiplier(Configuration conf) {
+ int blocksReplWorkMultiplier = conf.getInt(
+
DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
+
DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT);
+ Preconditions.checkArgument(
+ (blocksReplWorkMultiplier > 0),
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION +
+ " = '" + blocksReplWorkMultiplier + "' is invalid. " +
+ "It should be a positive, non-zero integer value.");
+ return blocksReplWorkMultiplier;
+ }
}
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1374645&r1=1374644&r2=1374645&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
Sat Aug 18 19:50:15 2012
@@ -268,23 +268,8 @@ public class BlockManager {
DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT);
this.shouldCheckForEnoughRacks =
conf.get(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY) != null;
- this.blocksInvalidateWorkPct = conf.getFloat(
- DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,
- DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT);
- Preconditions.checkArgument(
- (this.blocksInvalidateWorkPct > 0),
- DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION +
- " = '" + this.blocksInvalidateWorkPct + "' is invalid. " +
- "It should be a positive, non-zero float value " +
- "indicating a percentage.");
- this.blocksReplWorkMultiplier = conf.getInt(
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
-
DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT);
- Preconditions.checkArgument(
- (this.blocksReplWorkMultiplier > 0),
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION +
- " = '" + this.blocksReplWorkMultiplier + "' is invalid. " +
- "It should be a positive, non-zero integer value.");
+ this.blocksInvalidateWorkPct =
DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
this.replicationRecheckInterval =
conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1374645&r1=1374644&r2=1374645&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
Sat Aug 18 19:50:15 2012
@@ -42,7 +42,9 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
public class TestReplicationPolicy {
private Random random = DFSUtil.getRandom();
@@ -54,6 +56,9 @@ public class TestReplicationPolicy {
private static final String filename = "/dummyfile.txt";
private static DatanodeDescriptor dataNodes[];
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
@BeforeClass
public static void setupCluster() throws Exception {
Configuration conf = new HdfsConfiguration();
@@ -635,4 +640,51 @@ public class TestReplicationPolicy {
null, null, (short)2, first, second);
assertEquals(chosenNode, dataNodes[5]);
}
+
+ /**
+ * This testcase tests whether the defaultvalue returned by
+ * DFSUtil.getInvalidateWorkPctPerIteration() is positive,
+ * and whether an IllegalArgumentException will be thrown
+ * when a non-positive value is retrieved
+ */
+ @Test
+ public void testGetInvalidateWorkPctPerIteration() {
+ Configuration conf = new Configuration();
+ float blocksInvalidateWorkPct = DFSUtil
+ .getInvalidateWorkPctPerIteration(conf);
+ assertTrue(blocksInvalidateWorkPct > 0);
+
+ conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,
+ "0.5f");
+ blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ assertEquals(blocksInvalidateWorkPct, 0.5f, blocksInvalidateWorkPct *
1e-7);
+
+ conf.set(DFSConfigKeys.
+ DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "0.0");
+ exception.expect(IllegalArgumentException.class);
+ blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf);
+ }
+
+ /**
+ * This testcase tests whether the value returned by
+ * DFSUtil.getReplWorkMultiplier() is positive,
+ * and whether an IllegalArgumentException will be thrown
+ * when a non-positive value is retrieved
+ */
+ @Test
+ public void testGetReplWorkMultiplier() {
+ Configuration conf = new Configuration();
+ int blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+ assertTrue(blocksReplWorkMultiplier > 0);
+
+ conf.set(DFSConfigKeys.
+ DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,"3");
+ blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+ assertEquals(blocksReplWorkMultiplier, 3);
+
+ conf.set(DFSConfigKeys.
+ DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,"-1");
+ exception.expect(IllegalArgumentException.class);
+ blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
+ }
}
