Repository: sqoop Updated Branches: refs/heads/SQOOP-1367 f43835e7a -> 8f8977261 (forced update)
SQOOP-1236. Sqoop2: Classpath generated by Submission engine should contain only unique elements. (Jarcec via Hari) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/17c7219b Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/17c7219b Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/17c7219b Branch: refs/heads/SQOOP-1367 Commit: 17c7219b941240ddf21ecb52394d3dcfff846bd9 Parents: e60fda8 Author: Hari Shreedharan <[email protected]> Authored: Mon Jul 21 12:14:22 2014 -0700 Committer: Hari Shreedharan <[email protected]> Committed: Mon Jul 21 12:14:22 2014 -0700 ---------------------------------------------------------------------- .../sqoop/framework/SubmissionRequest.java | 10 ++- .../sqoop/framework/TestSubmissionRequest.java | 71 ++++++++++++++++++++ .../mapreduce/MapreduceSubmissionEngine.java | 14 +--- 3 files changed, 80 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/17c7219b/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java b/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java index 53d0039..a138db5 100644 --- a/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java +++ b/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java @@ -158,15 +158,19 @@ public class SubmissionRequest { } public void addJar(String jar) { - jars.add(jar); + if(!jars.contains(jar)) { + jars.add(jar); + } } public void addJarForClass(Class klass) { - jars.add(ClassUtils.jarForClass(klass)); + addJar(ClassUtils.jarForClass(klass)); } public void addJars(List<String> jars) { - this.jars.addAll(jars); + for(String j : jars) { + addJar(j); + } } public CallbackBase getConnectorCallbacks() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/17c7219b/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java b/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java new file mode 100644 index 0000000..3078ed2 --- /dev/null +++ b/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.framework; + +import org.apache.sqoop.utils.ClassUtils; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * + */ +public class TestSubmissionRequest { + + private SubmissionRequest submissionRequest; + + @Before + public void initializeSubmissionRequest() { + submissionRequest = new SubmissionRequest(); + } + + @Test + public void testAddJar() { + submissionRequest.addJar("A"); + submissionRequest.addJar("B"); + submissionRequest.addJar("A"); + + assertEquals(2, submissionRequest.getJars().size()); + assertEquals("A", submissionRequest.getJars().get(0)); + assertEquals("B", submissionRequest.getJars().get(1)); + } + + @Test + public void testAddJarForClass() { + submissionRequest.addJarForClass(TestSubmissionRequest.class); + submissionRequest.addJarForClass(TestFrameworkValidator.class); + + assertEquals(1, submissionRequest.getJars().size()); + assertTrue(submissionRequest.getJars().contains(ClassUtils.jarForClass(TestSubmissionRequest.class))); + } + + @Test + public void testAddJars() { + submissionRequest.addJars(Arrays.asList("A", "B")); + submissionRequest.addJars(Arrays.asList("B", "C")); + + assertEquals(3, submissionRequest.getJars().size()); + assertEquals("A", submissionRequest.getJars().get(0)); + assertEquals("B", submissionRequest.getJars().get(1)); + assertEquals("C", submissionRequest.getJars().get(2)); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/17c7219b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java ---------------------------------------------------------------------- diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java index 6fc485b..bfc28ef 100644 --- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java +++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java @@ -17,6 +17,7 @@ */ package org.apache.sqoop.submission.mapreduce; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobClient; @@ -187,18 +188,7 @@ public class MapreduceSubmissionEngine extends SubmissionEngine { configuration.setBoolean("mapred.reduce.tasks.speculative.execution", false); // Promote all required jars to the job - StringBuilder sb = new StringBuilder(); - boolean first = true; - for(String jar : request.getJars()) { - if(first) { - first = false; - } else { - sb.append(","); - } - LOG.debug("Adding jar to the job: " + jar); - sb.append(jar); - } - configuration.set("tmpjars", sb.toString()); + configuration.set("tmpjars", StringUtils.join(request.getJars(), ",")); try { Job job = new Job(configuration);
