Author: ekoifman
Date: Mon Mar 30 18:00:26 2015
New Revision: 1670160
URL: http://svn.apache.org/r1670160
Log:
HIVE-10050 Support overriding memory configuration for AM launched for
TempletonControllerJob (Hitesh Shah via Eugene Koifman)
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1670160&r1=1670159&r2=1670160&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Mon Mar
30 18:00:26 2015
@@ -197,6 +197,32 @@
</description>
</property>
+ <!--
+ <property>
+ <name>templeton.controller.mr.am.java.opts</name>
+ <value></value>
+ <description>Java options to be set for the templeton controller job's
+ MapReduce application master. When submitting the controller job,
+ Templeton will override yarn.app.mapreduce.am.command-opts with
+ this value. If this is not specified, Templeton will not set the
+ property and therefore the value will be picked up from
+ mapred-site.xml.
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.mr.am.memory.mb</name>
+ <value></value>
+ <description>Templeton controller job's Application Master's memory
+ limit in MB. When submitting controller job, Templeton will
+ overwrite yarn.app.mapreduce.am.resource.mb with this value. If
+ empty, Templeton will not set yarn.app.mapreduce.am.resource.mb
+ when submitting the controller job, therefore the configuration
+ in mapred-site.xml will be used.
+ </description>
+ </property>
+ -->
+
<property>
<name>templeton.exec.envs</name>
<value>HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME</value>
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1670160&r1=1670159&r2=1670160&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
Mon Mar 30 18:00:26 2015
@@ -35,7 +35,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.SystemVariables;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hive.hcatalog.templeton.tool.JobState;
@@ -104,6 +103,8 @@ public class AppConfig extends Configura
public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive";
public static final String HIVE_PATH_NAME = "templeton.hive.path";
public static final String MAPPER_MEMORY_MB =
"templeton.mapper.memory.mb";
+ public static final String MR_AM_MEMORY_MB = "templeton.mr.am.memory.mb";
+
/**
* see webhcat-default.xml
*/
@@ -130,6 +131,8 @@ public class AppConfig extends Configura
public static final String OVERRIDE_JARS_ENABLED =
"templeton.override.enabled";
public static final String TEMPLETON_CONTROLLER_MR_CHILD_OPTS
= "templeton.controller.mr.child.opts";
+ public static final String TEMPLETON_CONTROLLER_MR_AM_JAVA_OPTS
+ = "templeton.controller.mr.am.java.opts";
public static final String KERBEROS_SECRET = "templeton.kerberos.secret";
public static final String KERBEROS_PRINCIPAL =
"templeton.kerberos.principal";
@@ -148,6 +151,8 @@ public class AppConfig extends Configura
= "mapred.map.tasks.speculative.execution";
public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts";
public static final String HADOOP_MAP_MEMORY_MB = "mapreduce.map.memory.mb";
+ public static final String HADOOP_MR_AM_JAVA_OPTS =
"yarn.app.mapreduce.am.command-opts";
+ public static final String HADOOP_MR_AM_MEMORY_MB =
"yarn.app.mapreduce.am.resource.mb";
public static final String UNIT_TEST_MODE = "templeton.unit.test.mode";
@@ -313,7 +318,13 @@ public class AppConfig extends Configura
public String controllerMRChildOpts() {
return get(TEMPLETON_CONTROLLER_MR_CHILD_OPTS);
}
+ public String controllerAMChildOpts() {
+ return get(TEMPLETON_CONTROLLER_MR_AM_JAVA_OPTS);
+ }
public String mapperMemoryMb() { return get(MAPPER_MEMORY_MB); }
+ public String amMemoryMb() {
+ return get(MR_AM_MEMORY_MB);
+ }
/**
* @see #HIVE_PROPS_NAME
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1670160&r1=1670159&r2=1670160&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java
Mon Mar 30 18:00:26 2015
@@ -18,9 +18,7 @@
*/
package org.apache.hive.hcatalog.templeton.tool;
-import java.io.File;
import java.io.IOException;
-import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
@@ -28,13 +26,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
@@ -47,7 +41,6 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Tool;
import org.apache.hive.hcatalog.templeton.AppConfig;
-import org.apache.hive.hcatalog.templeton.Main;
import org.apache.hive.hcatalog.templeton.SecureProxySupport;
import org.apache.hive.hcatalog.templeton.UgiFactory;
import org.apache.thrift.TException;
@@ -114,6 +107,15 @@ public class TempletonControllerJob exte
if(memoryMb != null && memoryMb.length() != 0) {
conf.set(AppConfig.HADOOP_MAP_MEMORY_MB, memoryMb);
}
+ String amMemoryMB = appConf.amMemoryMb();
+ if (amMemoryMB != null && !amMemoryMB.isEmpty()) {
+ conf.set(AppConfig.HADOOP_MR_AM_MEMORY_MB, amMemoryMB);
+ }
+ String amJavaOpts = appConf.controllerAMChildOpts();
+ if (amJavaOpts != null && !amJavaOpts.isEmpty()) {
+ conf.set(AppConfig.HADOOP_MR_AM_JAVA_OPTS, amJavaOpts);
+ }
+
String user = UserGroupInformation.getCurrentUser().getShortUserName();
conf.set("user.name", user);
Job job = new Job(conf);
