Revert "MAPREDUCE-5785. Derive heap size or mapreduce.*.memory.mb automatically. (Gera Shegalov and Karthik Kambatla via kasha)"
This reverts commit a4df9eed059977374c8e889cb85d79e8e514ad30. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a655973e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a655973e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a655973e Branch: refs/heads/HDFS-EC Commit: a655973e781caf662b360c96e0fa3f5a873cf676 Parents: fe1f2db Author: Karthik Kambatla <ka...@apache.org> Authored: Tue Nov 25 16:24:26 2014 -0800 Committer: Karthik Kambatla <ka...@apache.org> Committed: Tue Nov 25 16:24:26 2014 -0800 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 - .../apache/hadoop/mapred/MapReduceChildJVM.java | 32 +++- .../v2/app/job/impl/TaskAttemptImpl.java | 20 ++- .../v2/app/job/impl/TestMapReduceChildJVM.java | 151 +++++-------------- .../java/org/apache/hadoop/mapred/JobConf.java | 125 +-------------- .../java/org/apache/hadoop/mapred/Task.java | 6 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 5 - .../src/main/resources/mapred-default.xml | 38 +---- 8 files changed, 93 insertions(+), 287 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a655973e/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 912f1a9..35efe7f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -235,9 +235,6 @@ Release 2.7.0 - UNRELEASED IMPROVEMENTS - MAPREDUCE-5785. Derive heap size or mapreduce.*.memory.mb automatically. - (Gera Shegalov and Karthik Kambatla via kasha) - OPTIMIZATIONS MAPREDUCE-6169. MergeQueue should release reference to the current item http://git-wip-us.apache.org/repos/asf/hadoop/blob/a655973e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java index bf0ceb9..c790c57 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java @@ -30,7 +30,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.TaskLog.LogName; import org.apache.hadoop.mapreduce.ID; import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; @@ -116,7 +115,36 @@ public class MapReduceChildJVM { } private static String getChildJavaOpts(JobConf jobConf, boolean isMapTask) { - return jobConf.getTaskJavaOpts(isMapTask ? TaskType.MAP : TaskType.REDUCE); + String userClasspath = ""; + String adminClasspath = ""; + if (isMapTask) { + userClasspath = + jobConf.get( + JobConf.MAPRED_MAP_TASK_JAVA_OPTS, + jobConf.get( + JobConf.MAPRED_TASK_JAVA_OPTS, + JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS) + ); + adminClasspath = + jobConf.get( + MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, + MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS); + } else { + userClasspath = + jobConf.get( + JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, + jobConf.get( + JobConf.MAPRED_TASK_JAVA_OPTS, + JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS) + ); + adminClasspath = + jobConf.get( + MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, + MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS); + } + + // Add admin classpath first so it can be overridden by user. + return adminClasspath + " " + userClasspath; } private static void setupLog4jProperties(Task task, http://git-wip-us.apache.org/repos/asf/hadoop/blob/a655973e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 3f54a85..dfc6a3f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -564,14 +564,18 @@ public abstract class TaskAttemptImpl implements } private int getMemoryRequired(Configuration conf, TaskType taskType) { - JobConf jobConf = conf instanceof JobConf - ? (JobConf) conf - : new JobConf(conf); - - return jobConf.getMemoryRequired( - taskType == TaskType.MAP - ? org.apache.hadoop.mapreduce.TaskType.MAP - : org.apache.hadoop.mapreduce.TaskType.REDUCE); + int memory = 1024; + if (taskType == TaskType.MAP) { + memory = + conf.getInt(MRJobConfig.MAP_MEMORY_MB, + MRJobConfig.DEFAULT_MAP_MEMORY_MB); + } else if (taskType == TaskType.REDUCE) { + memory = + conf.getInt(MRJobConfig.REDUCE_MEMORY_MB, + MRJobConfig.DEFAULT_REDUCE_MEMORY_MB); + } + + return memory; } private int getCpuRequired(Configuration conf, TaskType taskType) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a655973e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java index 9aa4dbb..8e146b9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java @@ -20,11 +20,8 @@ package org.apache.hadoop.mapreduce.v2.app.job.impl; import java.util.Map; -import org.apache.hadoop.mapreduce.TaskType; import org.junit.Assert; -import java.util.ArrayList; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -46,20 +43,6 @@ public class TestMapReduceChildJVM { private static final Log LOG = LogFactory.getLog(TestMapReduceChildJVM.class); - private final String[] expectedContents = { - "[", MRApps.crossPlatformify("JAVA_HOME") + "/bin/java", - "-Djava.net.preferIPv4Stack=true", - "-Dhadoop.metrics.log.level=WARN", - "-Xmx820m", - "-Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp", - "-Dyarn.app.container.log.dir=<LOG_DIR>", - "-Dyarn.app.container.log.filesize=0", - "-Dhadoop.root.logger=INFO,CLA", - "org.apache.hadoop.mapred.YarnChild", - "127.0.0.1", "54321", "attempt_0_0000_m_000000_0", - "0", "1><LOG_DIR>/stdout", - "2><LOG_DIR>/stderr ]"}; - @Test (timeout = 30000) public void testCommandLine() throws Exception { @@ -70,16 +53,22 @@ public class TestMapReduceChildJVM { app.waitForState(job, JobState.SUCCEEDED); app.verifyCompleted(); - for (String content : expectedContents) { - Assert.assertTrue("Missing argument", - app.launchCmdList.get(0).contains(content)); - } - - // Check log4j - Assert.assertTrue("Missing argument", - app.launchCmdList.get(0).contains( - "-Dlog4j.configuration=container-log4j.properties")); - + Assert.assertEquals( + "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + + " -Djava.net.preferIPv4Stack=true" + + " -Dhadoop.metrics.log.level=WARN" + + " -Xmx200m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + + " -Dlog4j.configuration=container-log4j.properties" + + " -Dyarn.app.container.log.dir=<LOG_DIR>" + + " -Dyarn.app.container.log.filesize=0" + + " -Dhadoop.root.logger=INFO,CLA" + + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + + " 54321" + + " attempt_0_0000_m_000000_0" + + " 0" + + " 1><LOG_DIR>/stdout" + + " 2><LOG_DIR>/stderr ]", app.myCommandLine); + Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER")); Assert.assertEquals("INFO,console", @@ -95,99 +84,33 @@ public class TestMapReduceChildJVM { MyMRApp app = new MyMRApp(1, 0, true, this.getClass().getName(), true); Configuration conf = new Configuration(); conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true); - String testLogPropertyFile = "test-log4j.properties"; - String testLogPropertyPath = "../"+"test-log4j.properties"; - conf.set(MRJobConfig.MAPREDUCE_JOB_LOG4J_PROPERTIES_FILE, testLogPropertyPath); + String testLogPropertieFile = "test-log4j.properties"; + String testLogPropertiePath = "../"+"test-log4j.properties"; + conf.set(MRJobConfig.MAPREDUCE_JOB_LOG4J_PROPERTIES_FILE, testLogPropertiePath); Job job = app.submit(conf); app.waitForState(job, JobState.SUCCEEDED); app.verifyCompleted(); - for (String content : expectedContents) { - Assert.assertTrue("Missing argument", - app.launchCmdList.get(0).contains(content)); - } - - // Check log4j - Assert.assertTrue("Missing argument", - app.launchCmdList.get(0).contains( - "-Dlog4j.configuration=" + testLogPropertyFile)); - } - - @Test - public void testAutoHeapSizes() throws Exception { - // Don't specify heap size or memory-mb - testAutoHeapSize(-1, -1, null); - - // Don't specify heap size - testAutoHeapSize(512, 768, null); - testAutoHeapSize(100, 768, null); - testAutoHeapSize(512, 100, null); - - // Specify heap size - testAutoHeapSize(512, 768, "-Xmx100m"); - testAutoHeapSize(512, 768, "-Xmx500m"); - - // Specify heap size but not the memory - testAutoHeapSize(-1, -1, "-Xmx100m"); - testAutoHeapSize(-1, -1, "-Xmx500m"); - } - - private void testAutoHeapSize(int mapMb, int redMb, String xmxArg) - throws Exception { - JobConf conf = new JobConf(new Configuration()); - float heapRatio = conf.getFloat(MRJobConfig.HEAP_MEMORY_MB_RATIO, - MRJobConfig.DEFAULT_HEAP_MEMORY_MB_RATIO); - - // Verify map and reduce java opts are not set by default - Assert.assertNull("Default map java opts!", - conf.get(MRJobConfig.MAP_JAVA_OPTS)); - Assert.assertNull("Default reduce java opts!", - conf.get(MRJobConfig.REDUCE_JAVA_OPTS)); - - // Set the memory-mbs and java-opts - if (mapMb > 0) { - conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMb); - } else { - mapMb = conf.getMemoryRequired(TaskType.MAP); - } - - if (redMb > 0) { - conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, redMb); - } else { - redMb = conf.getMemoryRequired(TaskType.REDUCE); - } - - if (xmxArg != null) { - conf.set(MRJobConfig.MAP_JAVA_OPTS, xmxArg); - conf.set(MRJobConfig.REDUCE_JAVA_OPTS, xmxArg); - } - - // Submit job to let unspecified fields be picked up - MyMRApp app = new MyMRApp(1, 1, true, this.getClass().getName(), true); - Job job = app.submit(conf); - app.waitForState(job, JobState.SUCCEEDED); - app.verifyCompleted(); - - // Go through the tasks and verify the values are as expected - for (String cmd : app.launchCmdList) { - final boolean isMap = cmd.contains("_m_"); - int heapMb; - if (xmxArg == null) { - heapMb = (int)(Math.ceil((isMap ? mapMb : redMb) * heapRatio)); - } else { - final String javaOpts = conf.get(isMap - ? MRJobConfig.MAP_JAVA_OPTS - : MRJobConfig.REDUCE_JAVA_OPTS); - heapMb = JobConf.parseMaximumHeapSizeMB(javaOpts); - } - - Assert.assertEquals("Incorrect heapsize in the command opts", - heapMb, JobConf.parseMaximumHeapSizeMB(cmd)); - } + Assert.assertEquals( + "[" + MRApps.crossPlatformify("JAVA_HOME") + "/bin/java" + + " -Djava.net.preferIPv4Stack=true" + + " -Dhadoop.metrics.log.level=WARN" + + " -Xmx200m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" + + " -Dlog4j.configuration=" + testLogPropertieFile + + " -Dyarn.app.container.log.dir=<LOG_DIR>" + + " -Dyarn.app.container.log.filesize=0" + + " -Dhadoop.root.logger=INFO,CLA" + + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + + " 54321" + + " attempt_0_0000_m_000000_0" + + " 0" + + " 1><LOG_DIR>/stdout" + + " 2><LOG_DIR>/stderr ]", app.myCommandLine); } private static final class MyMRApp extends MRApp { - final ArrayList<String> launchCmdList = new ArrayList<String>(); + + private String myCommandLine; private Map<String, String> cmdEnvironment; public MyMRApp(int maps, int reduces, boolean autoComplete, @@ -206,7 +129,7 @@ public class TestMapReduceChildJVM { launchEvent.getContainerLaunchContext(); String cmdString = launchContext.getCommands().toString(); LOG.info("launchContext " + cmdString); - launchCmdList.add(cmdString); + myCommandLine = cmdString; cmdEnvironment = launchContext.getEnvironment(); } super.handle(event); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a655973e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index 4608505..03f1160 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -20,10 +20,8 @@ package org.apache.hadoop.mapred; import java.io.IOException; -import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -46,7 +44,6 @@ import org.apache.hadoop.mapred.lib.KeyFieldBasedComparator; import org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.util.ConfigUtil; import org.apache.hadoop.security.Credentials; @@ -117,8 +114,6 @@ import org.apache.log4j.Level; public class JobConf extends Configuration { private static final Log LOG = LogFactory.getLog(JobConf.class); - private static final Pattern JAVA_OPTS_XMX_PATTERN = - Pattern.compile(".*(?:^|\\s)-Xmx(\\d+)([gGmMkK]?)(?:$|\\s).*"); static{ ConfigUtil.loadResources(); @@ -252,7 +247,9 @@ public class JobConf extends Configuration { */ public static final String MAPRED_REDUCE_TASK_JAVA_OPTS = JobContext.REDUCE_JAVA_OPTS; - + + public static final String DEFAULT_MAPRED_TASK_JAVA_OPTS = "-Xmx200m"; + /** * @deprecated * Configuration key to set the maximum virtual memory available to the child @@ -2025,121 +2022,7 @@ public class JobConf extends Configuration { LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_REDUCE_TASK_ULIMIT)); } } - - private String getConfiguredTaskJavaOpts(TaskType taskType) { - String userClasspath = ""; - String adminClasspath = ""; - if (taskType == TaskType.MAP) { - userClasspath = get(JobConf.MAPRED_MAP_TASK_JAVA_OPTS, - get(JobConf.MAPRED_TASK_JAVA_OPTS)); - adminClasspath = get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, - MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS); - } else { - userClasspath = get(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, - get(JobConf.MAPRED_TASK_JAVA_OPTS)); - adminClasspath = get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, - MRJobConfig.DEFAULT_MAPRED_ADMIN_JAVA_OPTS); - } - - // Add admin classpath first so it can be overridden by user. - return adminClasspath + " " + userClasspath; - } - - @Private - public String getTaskJavaOpts(TaskType taskType) { - String javaOpts = getConfiguredTaskJavaOpts(taskType); - - if (!javaOpts.contains("-Xmx")) { - float heapRatio = getFloat(MRJobConfig.HEAP_MEMORY_MB_RATIO, - MRJobConfig.DEFAULT_HEAP_MEMORY_MB_RATIO); - - if (heapRatio > 1.0f || heapRatio < 0) { - LOG.warn("Invalid value for " + MRJobConfig.HEAP_MEMORY_MB_RATIO - + ", using the default."); - heapRatio = MRJobConfig.DEFAULT_HEAP_MEMORY_MB_RATIO; - } - - int taskContainerMb = getMemoryRequired(taskType); - int taskHeapSize = (int)Math.ceil(taskContainerMb * heapRatio); - - String xmxArg = String.format("-Xmx%dm", taskHeapSize); - LOG.info("Task java-opts do not specify heap size. Setting task attempt" + - " jvm max heap size to " + xmxArg); - - javaOpts += " " + xmxArg; - } - - return javaOpts; - } - - /** - * Parse the Maximum heap size from the java opts as specified by the -Xmx option - * Format: -Xmx<size>[g|G|m|M|k|K] - * @param javaOpts String to parse to read maximum heap size - * @return Maximum heap size in MB or -1 if not specified - */ - @Private - @VisibleForTesting - public static int parseMaximumHeapSizeMB(String javaOpts) { - // Find the last matching -Xmx following word boundaries - Matcher m = JAVA_OPTS_XMX_PATTERN.matcher(javaOpts); - if (m.matches()) { - int size = Integer.parseInt(m.group(1)); - if (size <= 0) { - return -1; - } - if (m.group(2).isEmpty()) { - // -Xmx specified in bytes - return size / (1024 * 1024); - } - char unit = m.group(2).charAt(0); - switch (unit) { - case 'g': - case 'G': - // -Xmx specified in GB - return size * 1024; - case 'm': - case 'M': - // -Xmx specified in MB - return size; - case 'k': - case 'K': - // -Xmx specified in KB - return size / 1024; - } - } - // -Xmx not specified - return -1; - } - - @Private - public int getMemoryRequired(TaskType taskType) { - int memory = 1024; - int heapSize = parseMaximumHeapSizeMB(getConfiguredTaskJavaOpts(taskType)); - float heapRatio = getFloat(MRJobConfig.HEAP_MEMORY_MB_RATIO, - MRJobConfig.DEFAULT_HEAP_MEMORY_MB_RATIO); - if (taskType == TaskType.MAP) { - if (get(MRJobConfig.MAP_MEMORY_MB) == null && heapSize > 0) { - memory = (int) Math.ceil(heapSize / heapRatio); - LOG.info(MRJobConfig.MAP_MEMORY_MB + - " not specified. Derived from javaOpts = " + memory); - } else { - memory = getInt(MRJobConfig.MAP_MEMORY_MB, - MRJobConfig.DEFAULT_MAP_MEMORY_MB); - } - } else if (taskType == TaskType.REDUCE) { - if (get(MRJobConfig.REDUCE_MEMORY_MB) == null && heapSize > 0) { - memory = (int) Math.ceil(heapSize / heapRatio); - LOG.info(MRJobConfig.REDUCE_MEMORY_MB + - " not specified. Derived from javaOpts = " + memory); - } else { - memory = getInt(MRJobConfig.REDUCE_MEMORY_MB, - MRJobConfig.DEFAULT_REDUCE_MEMORY_MB); - } - } - - return memory; - } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a655973e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java index 5274438..3a4c513 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java @@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import javax.crypto.SecretKey; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -625,9 +624,8 @@ abstract public class Task implements Writable, Configurable { * Using AtomicBoolean since we need an atomic read & reset method. */ private AtomicBoolean progressFlag = new AtomicBoolean(false); - - @VisibleForTesting - public TaskReporter(Progress taskProgress, + + TaskReporter(Progress taskProgress, TaskUmbilicalProtocol umbilical) { this.umbilical = umbilical; this.taskProgress = taskProgress; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a655973e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 3ab29b0..691074a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -799,11 +799,6 @@ public interface MRJobConfig { public static final String TASK_PREEMPTION = "mapreduce.job.preemption"; - public static final String HEAP_MEMORY_MB_RATIO = - "mapreduce.job.heap.memory-mb.ratio"; - - public static final float DEFAULT_HEAP_MEMORY_MB_RATIO = 0.8f; - public static final String MR_ENCRYPTED_INTERMEDIATE_DATA = "mapreduce.job.encrypted-intermediate-data"; public static final boolean DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a655973e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 6f43c6c..6be62ec 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -208,11 +208,9 @@ <property> <name>mapreduce.map.memory.mb</name> - <!--value>1024</value--> + <value>1024</value> <description>The amount of memory to request from the scheduler for each - map task. If this is not specified, it is inferred from - mapreduce.map.java.opts and mapreduce.job.heap.memory-mb.ratio. - If java-opts are also not specified, we set it to 1024. + map task. </description> </property> @@ -226,11 +224,9 @@ <property> <name>mapreduce.reduce.memory.mb</name> - <!--value>1024</value--> + <value>1024</value> <description>The amount of memory to request from the scheduler for each - reduce task. If this is not specified, it is inferred from - mapreduce.reduce.java.opts and mapreduce.job.heap.memory-mb.ratio. - If java-opts are also not specified, we set it to 1024. + reduce task. </description> </property> @@ -244,7 +240,7 @@ <property> <name>mapred.child.java.opts</name> - <!--value></value--> + <value>-Xmx200m</value> <description>Java opts for the task processes. The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. @@ -255,10 +251,7 @@ Usage of -Djava.library.path can cause programs to no longer function if hadoop native libraries are used. These values should instead be set as part of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and - mapreduce.reduce.env config settings. - - If -Xmx is not set, it is inferred from mapreduce.{map|reduce}.memory.mb and - mapreduce.job.heap.memory-mb.ratio. + mapreduce.reduce.env config settings. </description> </property> @@ -267,9 +260,7 @@ <name>mapreduce.map.java.opts</name> <value></value> <description>Java opts only for the child processes that are maps. If set, - this will be used instead of mapred.child.java.opts. If -Xmx is not set, - it is inferred from mapreduce.map.memory.mb and - mapreduce.job.heap.memory-mb.ratio. + this will be used instead of mapred.child.java.opts. </description> </property> --> @@ -279,9 +270,7 @@ <name>mapreduce.reduce.java.opts</name> <value></value> <description>Java opts only for the child processes that are reduces. If set, - this will be used instead of mapred.child.java.opts. If -Xmx is not set, - it is inferred from mapreduce.reduce.memory.mb and - mapreduce.job.heap.memory-mb.ratio. + this will be used instead of mapred.child.java.opts. </description> </property> --> @@ -1534,15 +1523,4 @@ - HTTPS_ONLY : Service is provided only on https </description> </property> - -<property> - <name>mapreduce.job.heap.memory-mb.ratio</name> - <value>0.8</value> - <description>The ratio of heap-size to container-size. If no -Xmx is - specified, it is calculated as - (mapreduce.{map|reduce}.memory.mb * mapreduce.heap.memory-mb.ratio). - If -Xmx is specified but not mapreduce.{map|reduce}.memory.mb, it is - calculated as (heapSize / mapreduce.heap.memory-mb.ratio). - </description> -</property> </configuration>