Author: yhemanth
Date: Mon Oct 12 09:47:36 2009
New Revision: 824283
URL: http://svn.apache.org/viewvc?rev=824283&view=rev
Log:
MAPREDUCE-979. Fixed JobConf APIs related to memory parameters to return values
of new configuration variables when deprecated variables are disabled.
Contributed by Sreekanth Ramakrishnan.
Modified:
hadoop/common/branches/branch-0.20/CHANGES.txt
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/conf/TestJobConf.java
Modified: hadoop/common/branches/branch-0.20/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/CHANGES.txt?rev=824283&r1=824282&r2=824283&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20/CHANGES.txt Mon Oct 12 09:47:36 2009
@@ -13,6 +13,10 @@
MAPREDUCE-826. harchive doesn't use ToolRunner / harchive returns 0 even
if the job fails with exception (koji via mahadev)
+ MAPREDUCE-979. Fixed JobConf APIs related to memory parameters to return
+ values of new configuration variables when deprecated variables are
+ disabled. (Sreekanth Ramakrishnan via yhemanth)
+
Release 0.20.1 - 2009-09-01
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobConf.java?rev=824283&r1=824282&r2=824283&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobConf.java
(original)
+++
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobConf.java
Mon Oct 12 09:47:36 2009
@@ -112,7 +112,8 @@
}
/**
- * @deprecated
+ * @deprecated Use {...@link #MAPRED_JOB_MAP_MEMORY_MB_PROPERTY} and
+ * {...@link #MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY}
*/
@Deprecated
public static final String MAPRED_TASK_MAXVMEM_PROPERTY =
@@ -1427,32 +1428,68 @@
return get("job.local.dir");
}
+ /**
+ * Get memory required to run a map task of the job, in MB.
+ *
+ * If a value is specified in the configuration, it is returned.
+ * Else, it returns {...@link #DISABLED_MEMORY_LIMIT}.
+ * <p/>
+ * For backward compatibility, if the job configuration sets the
+ * key {...@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ * from {...@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ * after converting it from bytes to MB.
+ * @return memory required to run a map task of the job, in MB,
+ * or {...@link #DISABLED_MEMORY_LIMIT} if unset.
+ */
public long getMemoryForMapTask() {
- if (get(MAPRED_TASK_MAXVMEM_PROPERTY) != null) {
- long val = getLong(
- MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
- return (val == DISABLED_MEMORY_LIMIT) ? val :
- ((val < 0) ? DISABLED_MEMORY_LIMIT : val / (1024 * 1024));
+ long value = getDeprecatedMemoryValue();
+ if (value == DISABLED_MEMORY_LIMIT) {
+ value = normalizeMemoryConfigValue(
+ getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
+ DISABLED_MEMORY_LIMIT));
}
- return getLong(
- JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
- DISABLED_MEMORY_LIMIT);
+ return value;
}
public void setMemoryForMapTask(long mem) {
setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, mem);
}
+ /**
+ * Get memory required to run a reduce task of the job, in MB.
+ *
+ * If a value is specified in the configuration, it is returned.
+ * Else, it returns {...@link #DISABLED_MEMORY_LIMIT}.
+ * <p/>
+ * For backward compatibility, if the job configuration sets the
+ * key {...@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ * from {...@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ * after converting it from bytes to MB.
+ * @return memory required to run a reduce task of the job, in MB,
+ * or {...@link #DISABLED_MEMORY_LIMIT} if unset.
+ */
public long getMemoryForReduceTask() {
- if (get(MAPRED_TASK_MAXVMEM_PROPERTY) != null) {
- long val = getLong(
- MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
- return (val == DISABLED_MEMORY_LIMIT) ? val :
- ((val < 0) ? DISABLED_MEMORY_LIMIT : val / (1024 * 1024));
+ long value = getDeprecatedMemoryValue();
+ if (value == DISABLED_MEMORY_LIMIT) {
+ value = normalizeMemoryConfigValue(
+ getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
+ DISABLED_MEMORY_LIMIT));
+ }
+ return value;
+ }
+
+ // Return the value set to the key MAPRED_TASK_MAXVMEM_PROPERTY,
+ // converted into MBs.
+ // Returns DISABLED_MEMORY_LIMIT if unset, or set to a negative
+ // value.
+ private long getDeprecatedMemoryValue() {
+ long oldValue = getLong(MAPRED_TASK_MAXVMEM_PROPERTY,
+ DISABLED_MEMORY_LIMIT);
+ oldValue = normalizeMemoryConfigValue(oldValue);
+ if (oldValue != DISABLED_MEMORY_LIMIT) {
+ oldValue /= (1024*1024);
}
- return getLong(
- JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
- DISABLED_MEMORY_LIMIT);
+ return oldValue;
}
public void setMemoryForReduceTask(long mem) {
@@ -1524,18 +1561,21 @@
/**
- * The maximum amount of memory any task of this job will use. See
+ * Get the memory required to run a task of this job, in bytes. See
* {...@link #MAPRED_TASK_MAXVMEM_PROPERTY}
* <p/>
- * mapred.task.maxvmem is split into
- * mapred.job.map.memory.mb
- * and mapred.job.map.memory.mb,mapred
- * each of the new key are set
- * as mapred.task.maxvmem / 1024
- * as new values are in MB
- *
- * @return The maximum amount of memory any task of this job will use, in
- * bytes.
+ * This method is deprecated. Now, different memory limits can be
+ * set for map and reduce tasks of a job, in MB.
+ * <p/>
+ * For backward compatibility, if the job configuration sets the
+ * key {...@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ * from {...@link #DISABLED_MEMORY_LIMIT}, that value is returned.
+ * Otherwise, this method will return the larger of the values returned by
+ * {...@link #getMemoryForMapTask()} and {...@link #getMemoryForReduceTask()}
+ * after converting them into bytes.
+ *
+ * @return Memory required to run a task of this job, in bytes,
+ * or {...@link #DISABLED_MEMORY_LIMIT}, if unset.
* @see #setMaxVirtualMemoryForTask(long)
* @deprecated Use {...@link #getMemoryForMapTask()} and
* {...@link #getMemoryForReduceTask()}
@@ -1546,24 +1586,16 @@
"getMaxVirtualMemoryForTask() is deprecated. " +
"Instead use getMemoryForMapTask() and getMemoryForReduceTask()");
- if (get(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) == null) {
- if (get(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY) != null || get(
- JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY) != null) {
- long val = Math.max(getMemoryForMapTask(), getMemoryForReduceTask());
- if (val == JobConf.DISABLED_MEMORY_LIMIT) {
- return val;
- } else {
- if (val < 0) {
- return JobConf.DISABLED_MEMORY_LIMIT;
- }
- return val * 1024 * 1024;
- //Convert MB to byte as new value is in
- // MB and old deprecated method returns bytes
- }
+ long value = getLong(MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
+ value = normalizeMemoryConfigValue(value);
+ if (value == DISABLED_MEMORY_LIMIT) {
+ value = Math.max(getMemoryForMapTask(), getMemoryForReduceTask());
+ value = normalizeMemoryConfigValue(value);
+ if (value != DISABLED_MEMORY_LIMIT) {
+ value *= 1024*1024;
}
}
-
- return getLong(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY,
DISABLED_MEMORY_LIMIT);
+ return value;
}
/**
Modified:
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/conf/TestJobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/conf/TestJobConf.java?rev=824283&r1=824282&r2=824283&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/conf/TestJobConf.java
(original)
+++
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/conf/TestJobConf.java
Mon Oct 12 09:47:36 2009
@@ -70,9 +70,9 @@
configuration = new JobConf();
configuration.set("mapred.task.maxvmem" , "-1");
configuration.set("mapred.job.map.memory.mb",String.valueOf(300));
- configuration.set("mapred.job.reduce.memory.mb",String.valueOf(300));
- Assert.assertEquals(configuration.getMemoryForMapTask(),-1);
- Assert.assertEquals(configuration.getMemoryForReduceTask(),-1);
+ configuration.set("mapred.job.reduce.memory.mb",String.valueOf(400));
+ Assert.assertEquals(configuration.getMemoryForMapTask(), 300);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(), 400);
configuration = new JobConf();
configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
@@ -90,9 +90,47 @@
configuration = new JobConf();
configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+ configuration.set("mapred.job.map.memory.mb","3");
+ configuration.set("mapred.job.reduce.memory.mb","3");
Assert.assertEquals(configuration.getMemoryForMapTask(),2);
Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
}
+
+ /**
+ * Test that negative values for MAPRED_TASK_MAXVMEM_PROPERTY cause
+ * new configuration keys' values to be used.
+ */
+
+ public void testNegativeValueForTaskVmem() {
+ JobConf configuration = new JobConf();
+
+ configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3");
+ configuration.set("mapred.job.map.memory.mb", "4");
+ configuration.set("mapred.job.reduce.memory.mb", "5");
+ Assert.assertEquals(4, configuration.getMemoryForMapTask());
+ Assert.assertEquals(5, configuration.getMemoryForReduceTask());
+
+ }
+
+ /**
+ * Test that negative values for all memory configuration properties causes
+ * APIs to disable memory limits
+ */
+
+ public void testNegativeValuesForMemoryParams() {
+ JobConf configuration = new JobConf();
+
+ configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-4");
+ configuration.set("mapred.job.map.memory.mb", "-5");
+ configuration.set("mapred.job.reduce.memory.mb", "-6");
+
+ Assert.assertEquals(JobConf.DISABLED_MEMORY_LIMIT,
+ configuration.getMemoryForMapTask());
+ Assert.assertEquals(JobConf.DISABLED_MEMORY_LIMIT,
+ configuration.getMemoryForReduceTask());
+ Assert.assertEquals(JobConf.DISABLED_MEMORY_LIMIT,
+ configuration.getMaxVirtualMemoryForTask());
+ }
/**
* Test deprecated accessor and mutator method for mapred.task.maxvmem
