Repository: sqoop Updated Branches: refs/heads/sqoop2 6f113c094 -> 0773c10f8
SQOOP-2632: Sqoop2: add partition class to FROM (Dian Fu via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0773c10f Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0773c10f Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0773c10f Branch: refs/heads/sqoop2 Commit: 0773c10f83d2714cd05ce37393ec968f940ad302 Parents: 6f113c0 Author: Jarek Jarcec Cecho <[email protected]> Authored: Fri Oct 23 13:21:30 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Fri Oct 23 13:21:30 2015 -0700 ---------------------------------------------------------------------- .../apache/sqoop/connector/jdbc/GenericJdbcConnector.java | 1 + .../java/org/apache/sqoop/connector/hdfs/HdfsConnector.java | 1 + .../java/org/apache/sqoop/connector/kite/KiteConnector.java | 1 + .../src/main/java/org/apache/sqoop/job/etl/From.java | 8 ++++++++ .../sqoop/execution/mapreduce/MapreduceExecutionEngine.java | 1 + .../src/main/java/org/apache/sqoop/job/MRJobConstants.java | 3 +++ 6 files changed, 15 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java index 113465a..2083244 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java @@ -35,6 +35,7 @@ public class GenericJdbcConnector extends SqoopConnector { private static final From FROM = new From( GenericJdbcFromInitializer.class, GenericJdbcPartitioner.class, + GenericJdbcPartition.class, GenericJdbcExtractor.class, GenericJdbcFromDestroyer.class); http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java index 7e7c022..ecd2b68 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java @@ -38,6 +38,7 @@ public class HdfsConnector extends SqoopConnector { private static final From FROM = new From( HdfsFromInitializer.class, HdfsPartitioner.class, + HdfsPartition.class, HdfsExtractor.class, HdfsFromDestroyer.class); http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java index ca860b1..87a5464 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java @@ -46,6 +46,7 @@ public class KiteConnector extends SqoopConnector { private static final From FROM = new From( KiteFromInitializer.class, KiteDatasetPartitioner.class, + KiteDatasetPartition.class, KiteExtractor.class, KiteFromDestroyer.class); http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java index e9d2abe..1d2a3ed 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java @@ -34,14 +34,17 @@ import org.apache.sqoop.classification.InterfaceStability; public class From extends Transferable { private Class<? extends Partitioner> partitioner; + private Class<? extends Partition> partition; private Class<? extends Extractor> extractor; public From(Class<? extends Initializer> initializer, Class<? extends Partitioner> partitioner, + Class<? extends Partition> partition, Class<? extends Extractor> extractor, Class<? extends Destroyer> destroyer) { super(initializer, destroyer); this.partitioner = partitioner; + this.partition = partition; this.extractor = extractor; } @@ -49,6 +52,10 @@ public class From extends Transferable { return partitioner; } + public Class<? extends Partition> getPartition() { + return partition; + } + public Class<? extends Extractor> getExtractor() { return extractor; } @@ -57,6 +64,7 @@ public class From extends Transferable { public String toString() { return "From{" + super.toString() + ", partitioner=" + partitioner.getName() + + ", partition=" + partition.getName() + ", extractor=" + extractor.getName() + '}'; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java index c8d210e..403f213 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java @@ -67,6 +67,7 @@ public class MapreduceExecutionEngine extends ExecutionEngine { To to = (To) mrJobRequest.getTo(); MutableMapContext context = mrJobRequest.getDriverContext(); context.setString(MRJobConstants.JOB_ETL_PARTITIONER, from.getPartitioner().getName()); + context.setString(MRJobConstants.JOB_ETL_PARTITION, from.getPartition().getName()); context.setString(MRJobConstants.JOB_ETL_EXTRACTOR, from.getExtractor().getName()); context.setString(MRJobConstants.JOB_ETL_LOADER, to.getLoader().getName()); context.setString(MRJobConstants.JOB_ETL_FROM_DESTROYER, from.getDestroyer().getName()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java index b7aa8c6..df767e6 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java @@ -31,6 +31,9 @@ public final class MRJobConstants extends Constants { public static final String JOB_ETL_PARTITIONER = PREFIX_JOB_CONFIG + "etl.partitioner"; + public static final String JOB_ETL_PARTITION = PREFIX_JOB_CONFIG + + "etl.partition"; + public static final String JOB_ETL_EXTRACTOR = PREFIX_JOB_CONFIG + "etl.extractor";
