MAPREDUCE-6343. JobConf.parseMaximumHeapSizeMB() fails to parse value greater than 2GB expressed in bytes. (Hao Xia via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51909232 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51909232 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51909232 Branch: refs/heads/HDFS-7240 Commit: 519092322dd1bf71984bef1393d8e082643408cd Parents: bc1bd7e Author: Karthik Kambatla <[email protected]> Authored: Tue Apr 28 14:05:26 2015 -0700 Committer: Karthik Kambatla <[email protected]> Committed: Tue Apr 28 14:05:26 2015 -0700 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/mapred/JobConf.java | 10 +++++----- .../java/org/apache/hadoop/mapred/TestJobConf.java | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/51909232/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 2090007..e9e1a4b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -188,6 +188,9 @@ Trunk (Unreleased) MAPREDUCE-6234. TestHighRamJob fails due to the change in MAPREDUCE-5785. (Masatake Iwasaki via kasha) + MAPREDUCE-6343. JobConf.parseMaximumHeapSizeMB() fails to parse value + greater than 2GB expressed in bytes. (Hao Xia via kasha) + BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by http://git-wip-us.apache.org/repos/asf/hadoop/blob/51909232/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index 9cac685..059593a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -2080,28 +2080,28 @@ public class JobConf extends Configuration { // Find the last matching -Xmx following word boundaries Matcher m = JAVA_OPTS_XMX_PATTERN.matcher(javaOpts); if (m.matches()) { - int size = Integer.parseInt(m.group(1)); + long size = Long.parseLong(m.group(1)); if (size <= 0) { return -1; } if (m.group(2).isEmpty()) { // -Xmx specified in bytes - return size / (1024 * 1024); + return (int) (size / (1024 * 1024)); } char unit = m.group(2).charAt(0); switch (unit) { case 'g': case 'G': // -Xmx specified in GB - return size * 1024; + return (int) (size * 1024); case 'm': case 'M': // -Xmx specified in MB - return size; + return (int) size; case 'k': case 'K': // -Xmx specified in KB - return size / 1024; + return (int) (size / 1024); } } // -Xmx not specified http://git-wip-us.apache.org/repos/asf/hadoop/blob/51909232/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java index 0612ade..57fa4e6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java @@ -361,4 +361,20 @@ public class TestJobConf { jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxReduceAttempts() ); } + + /** + * Test parsing various types of Java heap options. + */ + @Test + public void testParseMaximumHeapSizeMB() { + // happy cases + Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4294967296")); + Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4194304k")); + Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4096m")); + Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4g")); + + // sad cases + Assert.assertEquals(-1, JobConf.parseMaximumHeapSizeMB("-Xmx4?")); + Assert.assertEquals(-1, JobConf.parseMaximumHeapSizeMB("")); + } }
