This is an automated email from the ASF dual-hosted git repository. stevel pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 188bebb HADOOP-16018. DistCp won't reassemble chunks when blocks per chunk > 0. 188bebb is described below commit 188bebbe7eaec71433548dc47f5a9a995cbea15a Author: Kai Xie <gigi...@gmail.com> AuthorDate: Tue Jan 8 11:56:48 2019 +0000 HADOOP-16018. DistCp won't reassemble chunks when blocks per chunk > 0. Contributed by Kai Xie. --- .../org/apache/hadoop/tools/DistCpConstants.java | 4 +++ .../apache/hadoop/tools/DistCpOptionSwitch.java | 2 +- .../org/apache/hadoop/tools/TestDistCpOptions.java | 29 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java index 212256c..4946091 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java @@ -123,6 +123,10 @@ public final class DistCpConstants { public static final String CONF_LABEL_COPY_BUFFER_SIZE = "distcp.copy.buffer.size"; + /** DistCp Blocks Per Chunk: {@value}. */ + public static final String CONF_LABEL_BLOCKS_PER_CHUNK = + "distcp.blocks.per.chunk"; + /** * Constants for DistCp return code to shell / consumer of ToolRunner's run */ diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java index 3ce12b2..e57e413 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java @@ -192,7 +192,7 @@ public enum DistCpOptionSwitch { new Option("sizelimit", true, "(Deprecated!) Limit number of files " + "copied to <= n bytes")), - BLOCKS_PER_CHUNK("", + BLOCKS_PER_CHUNK(DistCpConstants.CONF_LABEL_BLOCKS_PER_CHUNK, new Option("blocksperchunk", true, "If set to a positive value, files" + "with more blocks than this value will be split into chunks of " + "<blocksperchunk> blocks to be transferred in parallel, and " diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java index dd8ec69..62a2e6d 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java @@ -20,6 +20,7 @@ package org.apache.hadoop.tools; import java.util.Collections; +import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Test; @@ -533,4 +534,32 @@ public class TestDistCpOptions { builder.withLogPath(logPath).withVerboseLog(true); Assert.assertTrue(builder.build().shouldVerboseLog()); } + + @Test + public void testAppendToConf() { + final int expectedBlocksPerChunk = 999; + final String expectedValForEmptyConfigKey = "VALUE_OF_EMPTY_CONFIG_KEY"; + + DistCpOptions options = new DistCpOptions.Builder( + Collections.singletonList( + new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")) + .withBlocksPerChunk(expectedBlocksPerChunk) + .build(); + + Configuration config = new Configuration(); + config.set("", expectedValForEmptyConfigKey); + + options.appendToConf(config); + Assert.assertEquals(expectedBlocksPerChunk, + config.getInt( + DistCpOptionSwitch + .BLOCKS_PER_CHUNK + .getConfigLabel(), 0)); + Assert.assertEquals( + "Some DistCpOptionSwitch's config label is empty! " + + "Pls ensure the config label is provided when apply to config, " + + "otherwise it may not be fetched properly", + expectedValForEmptyConfigKey, config.get("")); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org