Repository: hadoop Updated Branches: refs/heads/branch-1 689ca40fc -> ebec913e2
MAPREDUCE-6147. Support mapreduce.input.fileinputformat.split.maxsize. (Zhihai Xu via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebec913e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebec913e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebec913e Branch: refs/heads/branch-1 Commit: ebec913e20d0081e2e30e13817e9023565e0b920 Parents: 689ca40 Author: Karthik Kambatla <[email protected]> Authored: Mon Nov 3 15:55:22 2014 -0800 Committer: Karthik Kambatla <[email protected]> Committed: Mon Nov 3 15:55:22 2014 -0800 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ .../mapred/lib/CombineFileInputFormat.java | 20 ++++++++++++++++-- .../mapred/lib/TestCombineFileInputFormat.java | 22 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebec913e/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 904bbd5..a695be2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -61,6 +61,9 @@ Release 1.3.0 - unreleased HADOOP-10614. CBZip2InputStream is not threadsafe (Xiangrui Meng via Sandy Ryza) + MAPREDUCE-6147. Support mapreduce.input.fileinputformat.split.maxsize. + (Zhihai Xu via kasha) + BUG FIXES MAPREDUCE-4490. Fixed LinuxTaskController to re-initialize user log http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebec913e/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java ---------------------------------------------------------------------- diff --git a/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java b/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java index 825100f..4977a35 100644 --- a/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java +++ b/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java @@ -141,7 +141,23 @@ public abstract class CombineFileInputFormat<K, V> } return codec instanceof SplittableCompressionCodec; } - + + /** + * First get "mapred.max.split.size". + * If "mapred.max.split.size" is not set, + * then get "mapreduce.input.fileinputformat.split.maxsize". + * If "mapreduce.input.fileinputformat.split.maxsize" is not set, + * then return 0. + */ + long getConfiguredMaxSplitSize(JobConf job) { + long maxSize = job.getLong("mapred.max.split.size", -1L); + if (maxSize == -1L) { + maxSize = job.getLong("mapreduce.input.fileinputformat.split.maxsize", + 0); + } + return maxSize; + } + /** * default constructor */ @@ -171,7 +187,7 @@ public abstract class CombineFileInputFormat<K, V> if (maxSplitSize != 0) { maxSize = maxSplitSize; } else { - maxSize = job.getLong("mapred.max.split.size", 0); + maxSize = getConfiguredMaxSplitSize(job); } if (minSizeNode != 0 && maxSize != 0 && minSizeNode > maxSize) { throw new IOException("Minimum split size pernode " + minSizeNode + http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebec913e/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java b/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java index 0d9e766..4ef9902 100644 --- a/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java +++ b/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java @@ -898,6 +898,28 @@ public class TestCombineFileInputFormat extends TestCase{ } } + public void testGetConfiguredMaxSplitSize() throws Throwable { + JobConf conf = new JobConf(); + DummyInputFormat inFormat = new DummyInputFormat(); + // if both mapred.max.split.size and + // mapreduce.input.fileinputformat.split.maxsize are not set, return 0. + long maxSize = inFormat.getConfiguredMaxSplitSize(conf); + assertEquals(maxSize, 0L); + + // if only mapreduce.input.fileinputformat.split.maxsize is set, + // return the value of mapreduce.input.fileinputformat.split.maxsize. + conf.setLong("mapreduce.input.fileinputformat.split.maxsize", 100L); + maxSize = inFormat.getConfiguredMaxSplitSize(conf); + assertEquals(maxSize, 100L); + + // if both mapred.max.split.size and + // mapreduce.input.fileinputformat.split.maxsize are set, + // return the value of mapred.max.split.size. + conf.setLong("mapred.max.split.size", 1000L); + maxSize = inFormat.getConfiguredMaxSplitSize(conf); + assertEquals(maxSize, 1000L); + } + static class TestFilter implements PathFilter { private Path p;
