Repository: incubator-blur Updated Branches: refs/heads/master 2683f4bd2 -> 6b7f2b049
fix multiplier on csv import job Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/6b7f2b04 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/6b7f2b04 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/6b7f2b04 Branch: refs/heads/master Commit: 6b7f2b049884559c4ee4712bd96403ebee2356d3 Parents: 2683f4b Author: twilliams <[email protected]> Authored: Tue Oct 21 17:11:19 2014 -0400 Committer: twilliams <[email protected]> Committed: Tue Oct 21 17:11:19 2014 -0400 ---------------------------------------------------------------------- .../apache/blur/mapreduce/lib/CsvBlurDriver.java | 10 ++++++---- .../blur/mapreduce/lib/CsvBlurDriverTest.java | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6b7f2b04/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java index 6830e32..1814945 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/CsvBlurDriver.java @@ -214,10 +214,6 @@ public class CsvBlurDriver { int maxDocumentBufferSize = Integer.parseInt(cmd.getOptionValue("b")); BlurOutputFormat.setMaxDocumentBufferSize(job, maxDocumentBufferSize); } - if (cmd.hasOption("r")) { - int reducerMultiplier = Integer.parseInt(cmd.getOptionValue("r")); - BlurOutputFormat.setReducerMultiplier(job, reducerMultiplier); - } // processing the 'd' option Option[] options = cmd.getOptions(); for (Option option : options) { @@ -231,6 +227,12 @@ public class CsvBlurDriver { } } BlurOutputFormat.setupJob(job, tableDescriptor); + + if (cmd.hasOption("r")) { + int reducerMultiplier = Integer.parseInt(cmd.getOptionValue("r")); + BlurOutputFormat.setReducerMultiplier(job, reducerMultiplier); + } + BlurMapReduceUtil.addDependencyJars(job.getConfiguration(), Splitter.class); return job; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/6b7f2b04/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java index 6a673eb..fd6f6c9 100644 --- a/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java +++ b/blur-mapred-hadoop1/src/test/java/org/apache/blur/mapreduce/lib/CsvBlurDriverTest.java @@ -133,6 +133,24 @@ public class CsvBlurDriverTest { assertEquals(SnappyCodec.class.getName(), configuration.get(CsvBlurDriver.MAPRED_MAP_OUTPUT_COMPRESSION_CODEC)); } + @Test + public void multiplierParamShouldIncreaseReduceTasks() throws Exception { + Configuration configurationSetup = new Configuration(); + ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() { + @Override + public Iface getClient(String controllerConnectionStr) { + return getMockIface(); + } + }; + int multiplierParam = 10; + Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, "-c", "host:40010", "-d", "family1", "col1", + "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", "file:///tmp/test1", "-i", "file:///tmp/test2", + "-S", "-C", "1000000", "2000000", "-p", "SNAPPY", "-r", Integer.toString(multiplierParam)); + assertNotNull(job); + + assertEquals(multiplierParam * shardCount, job.getNumReduceTasks()); + } + protected Iface getMockIface() { InvocationHandler handler = new InvocationHandler() {
