Repository: hadoop Updated Branches: refs/heads/trunk b047bc727 -> 6c154abd3
MAPREDUCE-6751. Add debug log message when splitting is not possible due to unsplittable compression. (Peter Vary via rchiang) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c154abd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c154abd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c154abd Branch: refs/heads/trunk Commit: 6c154abd33279475315b5f7f78dc47f1b0aa7028 Parents: b047bc7 Author: Ray Chiang <rchi...@apache.org> Authored: Tue Aug 16 12:13:22 2016 -0700 Committer: Ray Chiang <rchi...@apache.org> Committed: Tue Aug 16 12:13:22 2016 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/mapred/FileInputFormat.java | 7 +++++++ .../hadoop/mapreduce/lib/input/CombineFileInputFormat.java | 4 ++++ .../apache/hadoop/mapreduce/lib/input/FileInputFormat.java | 7 +++++++ 3 files changed, 18 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c154abd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java index 2c58ebe..5803d60 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java @@ -369,6 +369,13 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> { splitHosts[0], splitHosts[1])); } } else { + if (LOG.isDebugEnabled()) { + // Log only if the file is big enough to be splitted + if (length > Math.min(file.getBlockSize(), minSize)) { + LOG.debug("File is not splittable so no parallelization " + + "is possible: " + file.getPath()); + } + } String[][] splitHosts = getSplitHostsAndCachedHosts(blkLocations,0,length,clusterMap); splits.add(makeSplit(path, 0, length, splitHosts[0], splitHosts[1])); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c154abd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java index b2b7656..8f9699e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java @@ -600,6 +600,10 @@ public abstract class CombineFileInputFormat<K, V> if (!isSplitable) { // if the file is not splitable, just create the one block with // full file length + if (LOG.isDebugEnabled()) { + LOG.debug("File is not splittable so no parallelization " + + "is possible: " + stat.getPath()); + } blocks = new OneBlockInfo[1]; fileSize = stat.getLen(); blocks[0] = new OneBlockInfo(stat.getPath(), 0, fileSize, http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c154abd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java index 0c5ede9..7ec882f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java @@ -422,6 +422,13 @@ public abstract class FileInputFormat<K, V> extends InputFormat<K, V> { blkLocations[blkIndex].getCachedHosts())); } } else { // not splitable + if (LOG.isDebugEnabled()) { + // Log only if the file is big enough to be splitted + if (length > Math.min(file.getBlockSize(), minSize)) { + LOG.debug("File is not splittable so no parallelization " + + "is possible: " + file.getPath()); + } + } splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(), blkLocations[0].getCachedHosts())); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
