HADOOP-15850. CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0. Contributed by Ted Yu.
Signed-off-by: Wei-Chiu Chuang <[email protected]> (cherry picked from commit e2cecb681e2aab8b7c5465719cac53dce407a64c) (cherry picked from commit a7dd244a4995cbf20feb054530fc5913ccf384ac) modified: hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java (cherry picked from commit 84726679f0441bfa0788ad5a5ac2bfa4353afedb) (cherry picked from commit 8360d15abc29e9ed5f7a2117ab6ac882248e8458) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54757006 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54757006 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54757006 Branch: refs/heads/branch-2-jhung-test Commit: 547570063e2e6597dbeb45c242915b2b3038c4e7 Parents: 645b311 Author: Wei-Chiu Chuang <[email protected]> Authored: Fri Oct 19 13:56:23 2018 -0700 Committer: Jonathan Hung <[email protected]> Committed: Fri Dec 14 16:03:55 2018 -0800 ---------------------------------------------------------------------- .../java/org/apache/hadoop/tools/mapred/CopyCommitter.java | 8 +++++++- .../org/apache/hadoop/tools/mapred/TestCopyCommitter.java | 4 ---- 2 files changed, 7 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/54757006/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 6ddaab9..2b32af1 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -68,6 +68,7 @@ public class CopyCommitter extends FileOutputCommitter { private boolean overwrite = false; private boolean targetPathExists = true; private boolean ignoreFailures = false; + private int blocksPerChunk = 0; /** * Create a output committer @@ -78,6 +79,9 @@ public class CopyCommitter extends FileOutputCommitter { */ public CopyCommitter(Path outputPath, TaskAttemptContext context) throws IOException { super(outputPath, context); + blocksPerChunk = context.getConfiguration().getInt( + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getConfigLabel(), 0); + LOG.debug("blocks per chunk " + blocksPerChunk); this.taskAttemptContext = context; } @@ -92,7 +96,9 @@ public class CopyCommitter extends FileOutputCommitter { ignoreFailures = conf.getBoolean( DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); - concatFileChunks(conf); + if (blocksPerChunk > 0) { + concatFileChunks(conf); + } super.commitJob(jobContext); http://git-wip-us.apache.org/repos/asf/hadoop/blob/54757006/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java index 2452d6f..ec5d8ed 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java @@ -81,10 +81,6 @@ public class TestCopyCommitter { @Before public void createMetaFolder() { config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta"); - // Unset listing file path since the config is shared by - // multiple tests, and some test doesn't set it, such as - // testNoCommitAction, but the distcp code will check it. - config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, ""); Path meta = new Path("/meta"); try { cluster.getFileSystem().mkdirs(meta); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
