Repository: sqoop Updated Branches: refs/heads/sqoop2 5abcdd4d6 -> 80662d6d4
SQOOP-1898: Change the API of getJars to use SET than LIST to avoid duplicate jars (Veena Basavaraj via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/80662d6d Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/80662d6d Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/80662d6d Branch: refs/heads/sqoop2 Commit: 80662d6d4590b19a905a6c0d51c4580f3219c1c5 Parents: 5abcdd4 Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Dec 15 08:28:51 2014 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Dec 15 08:28:51 2014 -0800 ---------------------------------------------------------------------- .../jdbc/GenericJdbcFromInitializer.java | 5 +++-- .../jdbc/GenericJdbcToInitializer.java | 5 +++-- .../connector/kafka/KafkaToInitializer.java | 6 ++--- .../sqoop/connector/kite/KiteToInitializer.java | 8 +++---- .../idf/CSVIntermediateDataFormat.java | 4 ++-- .../connector/idf/IntermediateDataFormat.java | 10 ++++----- .../org/apache/sqoop/driver/JobRequest.java | 10 ++++----- .../org/apache/sqoop/driver/TestJobRequest.java | 23 +++++++++++--------- .../org/apache/sqoop/job/etl/Initializer.java | 8 +++---- 9 files changed, 42 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java index adf03d9..ff42949 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcFromInitializer.java @@ -22,6 +22,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.LinkedList; import java.util.List; +import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; @@ -56,8 +57,8 @@ public class GenericJdbcFromInitializer extends Initializer<LinkConfiguration, F } @Override - public List<String> getJars(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) { - List<String> jars = new LinkedList<String>(); + public Set<String> getJars(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) { + Set<String> jars = super.getJars(context, linkConfig, fromJobConfig); jars.add(ClassUtils.jarForClass(linkConfig.linkConfig.jdbcDriver)); return jars; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java index 930cfec..bc720da 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java @@ -22,6 +22,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.LinkedList; import java.util.List; +import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; @@ -53,8 +54,8 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration, ToJ } @Override - public List<String> getJars(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) { - List<String> jars = new LinkedList<String>(); + public Set<String> getJars(InitializerContext context, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) { + Set<String> jars = super.getJars(context, linkConfig, toJobConfig); jars.add(ClassUtils.jarForClass(linkConfig.linkConfig.jdbcDriver)); return jars; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java b/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java index e1b065a..923d1aa 100644 --- a/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java +++ b/connector/connector-kafka/src/main/java/org/apache/sqoop/connector/kafka/KafkaToInitializer.java @@ -24,7 +24,7 @@ import org.apache.sqoop.job.etl.Initializer; import org.apache.sqoop.job.etl.InitializerContext; import org.apache.sqoop.utils.ClassUtils; -import java.util.List; +import java.util.Set; public class KafkaToInitializer extends Initializer<LinkConfiguration,ToJobConfiguration> { @@ -37,9 +37,9 @@ public class KafkaToInitializer extends Initializer<LinkConfiguration,ToJobConfi @Override - public List<String> getJars(InitializerContext context, LinkConfiguration + public Set<String> getJars(InitializerContext context, LinkConfiguration linkConfiguration, ToJobConfiguration toJobConfiguration) { - List<String> jars = super.getJars(context, linkConfiguration, toJobConfiguration); + Set<String> jars = super.getJars(context, linkConfiguration, toJobConfiguration); // Jars for Kafka, Scala and Yammer (required by Kafka) jars.add(ClassUtils.jarForClass("kafka.javaapi.producer.Producer")); jars.add(ClassUtils.jarForClass("scala.collection.immutable.StringLike")); http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java index 40b3abf..11233a8 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java @@ -28,7 +28,7 @@ import org.apache.sqoop.schema.NullSchema; import org.apache.sqoop.schema.Schema; import org.apache.sqoop.utils.ClassUtils; -import java.util.List; +import java.util.Set; /** * This class allows connector to define initialization work for execution. @@ -50,9 +50,9 @@ public class KiteToInitializer extends Initializer<LinkConfiguration, } @Override - public List<String> getJars(InitializerContext context, - LinkConfiguration linkConfig, ToJobConfiguration jobConfig) { - List<String> jars = super.getJars(context, linkConfig, jobConfig); + public Set<String> getJars(InitializerContext context, + LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) { + Set<String> jars = super.getJars(context, linkConfig, toJobConfig); jars.add(ClassUtils.jarForClass("org.kitesdk.data.Formats")); jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.databind.JsonNode")); jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.core.TreeNode")); http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java index 73691ae..7529676 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java @@ -372,9 +372,9 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat<String> { * {@inheritDoc} */ @Override - public List<String> getJars() { + public Set<String> getJars() { - List<String> jars = super.getJars(); + Set<String> jars = super.getJars(); // Add JODA classes for IDF date/time handling jars.add(ClassUtils.jarForClass(LocalDate.class)); jars.add(ClassUtils.jarForClass(LocalDateTime.class)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java index b8c8042..7ac44dc 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java @@ -23,8 +23,8 @@ import org.apache.sqoop.schema.Schema; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.LinkedList; -import java.util.List; +import java.util.HashSet; +import java.util.Set; /** * Abstract class representing a pluggable intermediate data format Sqoop @@ -138,10 +138,10 @@ public abstract class IntermediateDataFormat<T> { /** * Provide the external jars that the IDF depends on - * @return list of jars + * @return set of jars */ - public List<String> getJars() { - return new LinkedList<String>(); + public Set<String> getJars() { + return new HashSet<String>(); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/core/src/main/java/org/apache/sqoop/driver/JobRequest.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java index c9377a7..d2496bd 100644 --- a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java +++ b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java @@ -61,9 +61,9 @@ public class JobRequest { SqoopConnector toConnector; /** - * List of required local jars for the job + * Set of required local jars for the job */ - List<String> jars; + Set<String> jars; /** * From entity @@ -123,7 +123,7 @@ public class JobRequest { Class<? extends IntermediateDataFormat<?>> toIDF; public JobRequest() { - this.jars = new LinkedList<String>(); + this.jars = new HashSet<String>(); this.fromConnectorContext = new MutableMapContext(); this.toConnectorContext = new MutableMapContext(); this.driverContext = new MutableMapContext(); @@ -188,7 +188,7 @@ public class JobRequest { } } - public List<String> getJars() { + public Set<String> getJars() { return jars; } @@ -202,7 +202,7 @@ public class JobRequest { addJar(ClassUtils.jarForClass(klass)); } - public void addJars(List<String> jars) { + public void addJars(Set<String> jars) { for(String j : jars) { addJar(j); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java index 9eff8a4..3f36030 100644 --- a/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java +++ b/core/src/test/java/org/apache/sqoop/driver/TestJobRequest.java @@ -18,6 +18,8 @@ package org.apache.sqoop.driver; import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import org.apache.sqoop.driver.JobRequest; import org.apache.sqoop.utils.ClassUtils; @@ -27,9 +29,6 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -/** - * - */ public class TestJobRequest { private JobRequest jobRequest; @@ -46,8 +45,8 @@ public class TestJobRequest { jobRequest.addJar("A"); assertEquals(2, jobRequest.getJars().size()); - assertEquals("A", jobRequest.getJars().get(0)); - assertEquals("B", jobRequest.getJars().get(1)); + assertTrue(jobRequest.getJars().contains("A")); + assertTrue(jobRequest.getJars().contains("B")); } @Test @@ -59,12 +58,16 @@ public class TestJobRequest { @Test public void testAddJars() { - jobRequest.addJars(Arrays.asList("A", "B")); - jobRequest.addJars(Arrays.asList("B", "C")); + Set<String> set = new HashSet<String>(); + set.addAll(Arrays.asList("A", "B")); + jobRequest.addJars(set); + set = new HashSet<String>(); + set.addAll(Arrays.asList("B", "C")); + jobRequest.addJars(set); assertEquals(3, jobRequest.getJars().size()); - assertEquals("A", jobRequest.getJars().get(0)); - assertEquals("B", jobRequest.getJars().get(1)); - assertEquals("C", jobRequest.getJars().get(2)); + assertTrue(jobRequest.getJars().contains("A")); + assertTrue(jobRequest.getJars().contains("A")); + assertTrue(jobRequest.getJars().contains("C")); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/80662d6d/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java ---------------------------------------------------------------------- diff --git a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java index 4dd6d5b..7dd156e 100644 --- a/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java +++ b/spi/src/main/java/org/apache/sqoop/job/etl/Initializer.java @@ -17,8 +17,8 @@ */ package org.apache.sqoop.job.etl; -import java.util.LinkedList; -import java.util.List; +import java.util.HashSet; +import java.util.Set; import org.apache.sqoop.schema.NullSchema; import org.apache.sqoop.schema.Schema; @@ -53,9 +53,9 @@ public abstract class Initializer<LinkConfiguration, JobConfiguration> { * In case of the TO initializer this will represent the TO job configuration * @return */ - public List<String> getJars(InitializerContext context, LinkConfiguration linkConfiguration, + public Set<String> getJars(InitializerContext context, LinkConfiguration linkConfiguration, JobConfiguration jobConfiguration) { - return new LinkedList<String>(); + return new HashSet<String>(); } /**
