Author: ramya
Date: Thu Oct 27 21:11:16 2011
New Revision: 1190035
URL: http://svn.apache.org/viewvc?rev=1190035&view=rev
Log:
HADOOP-7728. Enable task memory management to be configurable in hadoop config
setup script. (ramya)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1190035&r1=1190034&r2=1190035&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Oct
27 21:11:16 2011
@@ -55,6 +55,9 @@ Trunk (unreleased changes)
HADOOP-7758. Make GlobFilter class public. (tucu)
+ HADOOP-7728. Enable task memory management to be configurable in hadoop
+ config setup script. (ramya)
+
BUGS
HADOOP-7606. Upgrade Jackson to version 1.7.1 to match the version required
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh?rev=1190035&r1=1190034&r2=1190035&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
Thu Oct 27 21:11:16 2011
@@ -55,6 +55,16 @@ usage: $0 <parameters>
--dfs-support-append=false|true Enable
append
--hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts' Setup
proxy users for hadoop
--hbase-user=hbase User
which hbase is running as. Defaults to hbase
+ --mapreduce-cluster-mapmemory-mb=memory Virtual
memory of a map slot for the MR framework. Defaults to -1
+ --mapreduce-cluster-reducememory-mb=memory Virtual
memory, of a reduce slot for the MR framework. Defaults to -1
+ --mapreduce-jobtracker-maxmapmemory-mb=memory Maximum
virtual memory of a single map task. Defaults to -1
+ This
value should be set to (mapreduce.cluster.mapmemory.mb *
mapreduce.tasktracker.map.tasks.maximum)
+ --mapreduce-jobtracker-maxreducememory-mb=memory Maximum
virtual memory of a single reduce task. Defaults to -1
+ This
value should be set to (mapreduce.cluster.reducememory.mb *
mapreduce.tasktracker.reduce.tasks.maximum)
+ --mapreduce-map-memory-mb=memory Virtual
memory of a single map slot for a job. Defaults to -1
+ This
value should be <= mapred.cluster.max.map.memory.mb
+ --mapreduce-reduce-memory-mb=memory Virtual
memory, of a single reduce slot for a job. Defaults to -1
+ This
value should be <= mapred.cluster.max.reduce.memory.mb
"
exit 1
}
@@ -199,6 +209,12 @@ OPTS=$(getopt \
-l 'hadoop-proxy-users:' \
-l 'dfs-support-append:' \
-l 'hbase-user:' \
+ -l 'mapreduce-cluster-mapmemory-mb:' \
+ -l 'mapreduce-cluster-reducememory-mb:' \
+ -l 'mapreduce-jobtracker-maxmapmemory-mb:' \
+ -l 'mapreduce-jobtracker-maxreducememory-mb:' \
+ -l 'mapreduce-map-memory-mb:' \
+ -l 'mapreduce-reduce-memory-mb:' \
-o 'h' \
-- "$@")
@@ -334,6 +350,30 @@ while true ; do
HBASE_USER=$2; shift 2
AUTOMATED=1
;;
+ --mapreduce-cluster-mapmemory-mb)
+ MAPREDUCE_CLUSTER_MAPMEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-cluster-reducememory-mb)
+ MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-jobtracker-maxmapmemory-mb)
+ MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-jobtracker-maxreducememory-mb)
+ MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-map-memory-mb)
+ MAPREDUCE_MAP_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-reduce-memory-mb)
+ MAPREDUCE_REDUCE_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
--)
shift ; break
;;
@@ -365,6 +405,12 @@ HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
HBASE_USER=${HBASE_USER:-hbase}
+MAPREDUCE_CLUSTER_MAPMEMORY_MB=${MAPREDUCE_CLUSTER_MAPMEMORY_MB:--1}
+MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB:--1}
+MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB:--1}
+MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB:--1}
+MAPREDUCE_MAP_MEMORY_MB=${MAPREDUCE_MAP_MEMORY_MB:--1}
+MAPREDUCE_REDUCE_MEMORY_MB=${MAPREDUCE_REDUCE_MEMORY_MB:--1}
KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml?rev=1190035&r1=1190034&r2=1190035&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
Thu Oct 27 21:11:16 2011
@@ -280,4 +280,34 @@
<name>mapred.jobtracker.retirejob.interval</name>
<value>0</value>
</property>
+
+ <property>
+ <name>mapreduce.cluster.mapmemory.mb</name>
+ <value>${MAPREDUCE_CLUSTER_MAPMEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.cluster.reducememory.mb</name>
+ <value>${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobtracker.maxmapmemory.mb</name>
+ <value>${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobtracker.maxreducememory.mb</name>
+ <value>${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>${MAPREDUCE_MAP_MEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>${MAPREDUCE_REDUCE_MEMORY_MB}</value>
+ </property>
</configuration>
