Repository: sqoop Updated Branches: refs/heads/sqoop2 0c06e8b0b -> 7a2285c3c
SQOOP-1998: Sqoop2: Kite Connector: Hive support (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/7a2285c3 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/7a2285c3 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/7a2285c3 Branch: refs/heads/sqoop2 Commit: 7a2285c3c329927e89c1d06ebe4340a20092e0d4 Parents: 0c06e8b Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Mar 19 18:24:21 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Mar 19 18:24:21 2015 -0700 ---------------------------------------------------------------------- connector/connector-kite/pom.xml | 12 +++++++++ .../connector/kite/KiteDatasetExecutor.java | 26 +++++++++++++++----- .../connector/kite/KiteDatasetPartitioner.java | 5 +++- .../sqoop/connector/kite/KiteExtractor.java | 4 ++- .../connector/kite/KiteFromInitializer.java | 24 ++++++++++++++++-- .../apache/sqoop/connector/kite/KiteLoader.java | 12 ++++----- .../sqoop/connector/kite/KiteToInitializer.java | 16 +++++++++++- .../kite/configuration/ConfigUtil.java | 8 ++++++ .../sqoop/connector/kite/TestKiteExecutor.java | 19 ++++++++++++-- .../connector/kite/TestKiteFromInitializer.java | 5 ++-- .../sqoop/connector/kite/TestKiteLoader.java | 2 +- dist/src/main/server/conf/catalina.properties | 4 +-- pom.xml | 21 +++++++++++++--- 13 files changed, 131 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/pom.xml ---------------------------------------------------------------------- diff --git a/connector/connector-kite/pom.xml b/connector/connector-kite/pom.xml index 11ec534..8593899 100644 --- a/connector/connector-kite/pom.xml +++ b/connector/connector-kite/pom.xml @@ -41,6 +41,14 @@ limitations under the License. <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-serde</artifactId> + </dependency> <!-- Testing specified modules --> <dependency> @@ -70,6 +78,10 @@ limitations under the License. <artifactId>kite-data-core</artifactId> </dependency> <dependency> + <groupId>org.kitesdk</groupId> + <artifactId>kite-data-hive</artifactId> + </dependency> + <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java index 883925b..db8d4e6 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java @@ -25,6 +25,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.common.FileFormat; import org.apache.sqoop.connector.common.AvroDataTypeUtil; +import org.apache.sqoop.connector.kite.configuration.ConfigUtil; +import org.apache.sqoop.connector.kite.configuration.LinkConfig; import org.apache.sqoop.connector.kite.util.KiteDataTypeUtil; import org.apache.sqoop.error.code.KiteConnectorError; import org.kitesdk.data.Dataset; @@ -147,7 +149,6 @@ public class KiteDatasetExecutor { FileSystemDataset<GenericRecord> update = Datasets.load(uri); if (dataset instanceof FileSystemDataset) { ((FileSystemDataset<GenericRecord>) dataset).merge(update); - // And let's completely drop the temporary dataset Datasets.delete(uri); } else { @@ -156,14 +157,15 @@ public class KiteDatasetExecutor { } } - private static final String TEMPORARY_DATASET_PREFIX = "/temp_"; + private static final String TEMPORARY_DATASET_PREFIX = "temp_"; /** * Workaround for managing temporary datasets. */ - public static String suggestTemporaryDatasetUri(String uri) { - if (uri.startsWith("dataset:hdfs:")) { - int pathStart = uri.lastIndexOf(":") + 1; + public static String suggestTemporaryDatasetUri(LinkConfig linkConfig, String uri) { + if (uri.startsWith("dataset:hdfs:") || uri.startsWith("dataset:hive:")) { + int pathStart = uri.indexOf(":") + 1; + pathStart = uri.indexOf(":", pathStart); int pathEnd = uri.lastIndexOf("?"); String[] uriParts = null; @@ -179,7 +181,19 @@ public class KiteDatasetExecutor { uriParts[0] = uri.substring(0, pathStart); // Add to path - uriParts[1] += TEMPORARY_DATASET_PREFIX + UUID.randomUUID().toString().replace("-", ""); + String temporaryDatasetName = TEMPORARY_DATASET_PREFIX + UUID.randomUUID().toString().replace("-", ""); + if (uri.startsWith("dataset:hive:")) { + if (uriParts[1].lastIndexOf("/") > -1) { + // Kite creates databases from namespace names in hive. + // Re-use entire path except for dataset name (table name in hive). + // Replace dataset name with temporary dataset name. + uriParts[1] = uriParts[1].substring(0, uriParts[1].lastIndexOf("/")) + "/" + temporaryDatasetName; + } else { + uriParts[1] = temporaryDatasetName; + } + } else { + uriParts[1] += "/" + temporaryDatasetName; + } return StringUtils.join(uriParts, ""); } else { http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java index 41fe7e1..e03bd60 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java @@ -18,6 +18,7 @@ package org.apache.sqoop.connector.kite; +import org.apache.sqoop.connector.kite.configuration.ConfigUtil; import org.apache.sqoop.connector.kite.configuration.FromJobConfiguration; import org.apache.sqoop.connector.kite.configuration.LinkConfiguration; import org.apache.sqoop.job.etl.Partition; @@ -41,7 +42,9 @@ public class KiteDatasetPartitioner extends Partitioner<LinkConfiguration, // There is no way to create partitions of an un-partitioned dataset. // TODO: SQOOP-1942 will create partitions, if dataset is partitioned. KiteDatasetPartition partition = new KiteDatasetPartition(); - partition.setUri(fromJobConfig.fromJobConfig.uri); + String uri = ConfigUtil.buildDatasetUri( + linkConfiguration.linkConfig, fromJobConfig.fromJobConfig.uri); + partition.setUri(uri); List<Partition> partitions = new LinkedList<Partition>(); partitions.add(partition); http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java index d4a8a77..d93f9b5 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java @@ -20,6 +20,7 @@ package org.apache.sqoop.connector.kite; import com.google.common.annotations.VisibleForTesting; import org.apache.avro.generic.GenericRecord; import org.apache.log4j.Logger; +import org.apache.sqoop.connector.kite.configuration.ConfigUtil; import org.apache.sqoop.connector.kite.configuration.FromJobConfiguration; import org.apache.sqoop.connector.kite.configuration.LinkConfiguration; import org.apache.sqoop.etl.io.DataWriter; @@ -48,7 +49,8 @@ public class KiteExtractor extends Extractor<LinkConfiguration, @Override public void extract(ExtractorContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig, KiteDatasetPartition partition) { - String uri = partition.getUri(); + String uri = ConfigUtil.buildDatasetUri( + linkConfig.linkConfig, partition.getUri()); LOG.info("Loading data from " + uri); KiteDatasetExecutor executor = getExecutor(uri); http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java index 292510a..06f18e7 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java @@ -19,6 +19,7 @@ package org.apache.sqoop.connector.kite; import org.apache.log4j.Logger; import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.connector.kite.configuration.ConfigUtil; import org.apache.sqoop.connector.kite.configuration.FromJobConfiguration; import org.apache.sqoop.connector.kite.configuration.LinkConfiguration; import org.apache.sqoop.connector.common.AvroDataTypeUtil; @@ -43,7 +44,10 @@ public class KiteFromInitializer extends Initializer<LinkConfiguration, @Override public void initialize(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) { - if (!Datasets.exists(fromJobConfig.fromJobConfig.uri)) { + String uri = ConfigUtil.buildDatasetUri( + linkConfig.linkConfig, fromJobConfig.fromJobConfig.uri); + LOG.debug("Constructed dataset URI: " + uri); + if (!Datasets.exists(uri)) { LOG.error("Dataset does not exist"); throw new SqoopException(KiteConnectorError.GENERIC_KITE_CONNECTOR_0002); } @@ -55,13 +59,29 @@ public class KiteFromInitializer extends Initializer<LinkConfiguration, jars.add(ClassUtils.jarForClass("org.kitesdk.data.Datasets")); jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.databind.JsonNode")); jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.core.TreeNode")); + if (fromJobConfig.fromJobConfig.uri.startsWith("dataset:hive")) { + // @TODO(Abe): Remove a deps that aren't used? + jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.conf.HiveConf")); + jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.serde2.SerDe")); + jars.add(ClassUtils.jarForClass("org.kitesdk.data.spi.hive.MetaStoreUtil")); + jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors")); + jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse")); + jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat")); + jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory")); + jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService")); + jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler")); + jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery")); + jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement")); + } return jars; } @Override public Schema getSchema(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) { - Dataset dataset = Datasets.load(fromJobConfig.fromJobConfig.uri); + String uri = ConfigUtil.buildDatasetUri( + linkConfig.linkConfig, fromJobConfig.fromJobConfig.uri); + Dataset dataset = Datasets.load(uri); org.apache.avro.Schema avroSchema = dataset.getDescriptor().getSchema(); return AvroDataTypeUtil.createSqoopSchema(avroSchema); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java index 57790a1..ca0a5c7 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java @@ -40,27 +40,27 @@ public class KiteLoader extends Loader<LinkConfiguration, ToJobConfiguration> { private long rowsWritten = 0; @VisibleForTesting - KiteDatasetExecutor getExecutor(String uri, Schema schema, + KiteDatasetExecutor getExecutor(LinkConfiguration linkConfiguration, String uri, Schema schema, FileFormat format) { // Note that instead of creating a dataset at destination, we create a // temporary dataset by every KiteLoader instance. They will be merged when // all data portions are written successfully. Unfortunately, KiteLoader is // not able to pass the temporary dataset uri to KiteToDestroyer. So we // delegate KiteDatasetExecutor to manage name convention for datasets. - uri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri); - LOG.info("Generated temporary dataset URI: " + uri); + uri = KiteDatasetExecutor.suggestTemporaryDatasetUri(linkConfiguration.linkConfig, uri); + LOG.info("Constructed temporary dataset URI: " + uri); Dataset<GenericRecord> dataset = KiteDatasetExecutor.createDataset(uri, schema, format); return new KiteDatasetExecutor(dataset); } @Override - public void load(LoaderContext context, LinkConfiguration linkConfig, + public void load(LoaderContext context, LinkConfiguration linkConfiguration, ToJobConfiguration toJobConfig) throws Exception { String uri = ConfigUtil.buildDatasetUri( - linkConfig.linkConfig, toJobConfig.toJobConfig); + linkConfiguration.linkConfig, toJobConfig.toJobConfig); KiteDatasetExecutor executor = getExecutor( - uri, context.getSchema(), toJobConfig.toJobConfig.fileFormat); + linkConfiguration, uri, context.getSchema(), toJobConfig.toJobConfig.fileFormat); LOG.info("Temporary dataset created."); DataReader reader = context.getDataReader(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java index 187ad21..39a2989 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java @@ -48,7 +48,7 @@ public class KiteToInitializer extends Initializer<LinkConfiguration, LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) { String uri = ConfigUtil.buildDatasetUri( linkConfig.linkConfig, toJobConfig.toJobConfig); - LOG.info("Generated final dataset URI: " + uri); + LOG.debug("Constructed dataset URI: " + uri); if (Datasets.exists(uri)) { LOG.error("Overwrite an existing dataset is not expected in new create mode."); throw new SqoopException(KiteConnectorError.GENERIC_KITE_CONNECTOR_0001); @@ -65,6 +65,20 @@ public class KiteToInitializer extends Initializer<LinkConfiguration, if (FileFormat.CSV.equals(toJobConfig.toJobConfig.fileFormat)) { jars.add(ClassUtils.jarForClass("au.com.bytecode.opencsv.CSVWriter")); } + if (toJobConfig.toJobConfig.uri.startsWith("dataset:hive")) { + // @TODO(Abe): Remove a deps that aren't used? + jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.conf.HiveConf")); + jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.serde2.SerDe")); + jars.add(ClassUtils.jarForClass("org.kitesdk.data.spi.hive.MetaStoreUtil")); + jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors")); + jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse")); + jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat")); + jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory")); + jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService")); + jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler")); + jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery")); + jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement")); + } return jars; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java index 3052a87..e63bccf 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java @@ -48,6 +48,14 @@ public class ConfigUtil { * Returns a dataset uri, including the filesystem location part, if it is * provided separated, */ + public static String buildDatasetUri(LinkConfig linkConfig, String uri) { + return buildDatasetUri(linkConfig.authority, uri); + } + + /** + * Returns a dataset uri, including the filesystem location part, if it is + * provided separated, + */ public static String buildDatasetUri(LinkConfig linkConfig, ToJobConfig toJobConfig) { return buildDatasetUri(linkConfig.authority, toJobConfig.uri); http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java index 7173c18..44721a8 100644 --- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java +++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java @@ -20,6 +20,7 @@ package org.apache.sqoop.connector.kite; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecordBuilder; +import org.apache.sqoop.connector.kite.configuration.LinkConfig; import org.kitesdk.data.Dataset; import org.kitesdk.data.DatasetDescriptor; import org.kitesdk.data.DatasetReader; @@ -150,14 +151,28 @@ public class TestKiteExecutor { @Test public void testSuggestTemporaryDatasetUri() { String uri = "dataset:hdfs:/tmp/sqoop/test"; - String suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri); + String suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri); assertTrue(suggestedUri.length() > uri.length()); assertTrue(suggestedUri.contains(uri)); uri = "dataset:hdfs://namenode:8020/tmp/sqoop/test"; - suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri); + suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri); assertTrue(suggestedUri.length() > uri.length()); assertTrue(suggestedUri.contains(uri)); + + String subURI = "dataset:hive://metastore:9083/tmp/sqoop"; + uri = "dataset:hive://metastore:9083/tmp/sqoop/test"; + suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri); + assertTrue(suggestedUri.length() > subURI.length()); + assertTrue(suggestedUri.contains(subURI)); + + String endURI = "auth:host=metastore&auth:port=9083"; + uri = "dataset:hive:tmp/sqoop?auth:host=metastore&auth:port=9083"; + subURI = "dataset:hive:tmp"; + suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri); + assertTrue(suggestedUri.length() > subURI.length()); + assertTrue(suggestedUri.contains(subURI)); + assertTrue(suggestedUri.endsWith(endURI)); } private static Schema createTwoFieldSchema() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java index a60b2fc..6df5d83 100644 --- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java +++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java @@ -20,6 +20,7 @@ package org.apache.sqoop.connector.kite; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.kite.configuration.FromJobConfiguration; +import org.apache.sqoop.connector.kite.configuration.LinkConfiguration; import org.kitesdk.data.Datasets; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -53,7 +54,7 @@ public class TestKiteFromInitializer extends PowerMockTestCase { when(Datasets.exists(jobConfig.fromJobConfig.uri)).thenReturn(true); // exercise - initializer.initialize(null, null, jobConfig); + initializer.initialize(null, new LinkConfiguration(), jobConfig); } @Test(expectedExceptions = SqoopException.class) @@ -64,7 +65,7 @@ public class TestKiteFromInitializer extends PowerMockTestCase { when(Datasets.exists(jobConfig.fromJobConfig.uri)).thenReturn(false); // exercise - initializer.initialize(null, null, jobConfig); + initializer.initialize(null, new LinkConfiguration(), jobConfig); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java index b18c91e..533b8c3 100644 --- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java +++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java @@ -46,7 +46,7 @@ public class TestKiteLoader { loader = new KiteLoader() { @Override - protected KiteDatasetExecutor getExecutor(String uri, Schema schema, + protected KiteDatasetExecutor getExecutor(LinkConfiguration linkConfiguration, String uri, Schema schema, FileFormat format) { return executorMock; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/dist/src/main/server/conf/catalina.properties ---------------------------------------------------------------------- diff --git a/dist/src/main/server/conf/catalina.properties b/dist/src/main/server/conf/catalina.properties index 260bfd6..5e201be 100644 --- a/dist/src/main/server/conf/catalina.properties +++ b/dist/src/main/server/conf/catalina.properties @@ -50,9 +50,9 @@ package.definition=sun.,java.,org.apache.catalina.,org.apache.coyote.,org.apache # Sqoop installation guide for more details. # # Example (based on default BigTop locations): -#common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,${catalina.home}/../lib/*.jar,/usr/lib/hadoop/*.jar,/usr/lib/hadoop/lib/*.jar,/usr/lib/hadoop-hdfs/*.jar,/usr/lib/hadoop-hdfs/lib/*.jar,/usr/lib/hadoop-mapreduce/*.jar,/usr/lib/hadoop-mapreduce/lib/*.jar,/usr/lib/hadoop-yarn/*.jar,/usr/lib/hadoop-yarn/lib/*.jar +#common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,${catalina.home}/../lib/*.jar,/usr/lib/hadoop/*.jar,/usr/lib/hadoop/lib/*.jar,/usr/lib/hadoop-hdfs/*.jar,/usr/lib/hadoop-hdfs/lib/*.jar,/usr/lib/hadoop-mapreduce/*.jar,/usr/lib/hadoop-mapreduce/lib/*.jar,/usr/lib/hadoop-yarn/*.jar,/usr/lib/hadoop-yarn/lib/*.jar,/usr/lib/hive/lib/*.jar # -common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,${catalina.home}/../lib/*.jar,/usr/lib/hadoop/*.jar,/usr/lib/hadoop/lib/*.jar,/usr/lib/hadoop-hdfs/*.jar,/usr/lib/hadoop-hdfs/lib/*.jar,/usr/lib/hadoop-mapreduce/*.jar,/usr/lib/hadoop-mapreduce/lib/*.jar,/usr/lib/hadoop-yarn/*.jar,/usr/lib/hadoop-yarn/lib/*.jar +common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,${catalina.home}/../lib/*.jar,/usr/lib/hadoop/*.jar,/usr/lib/hadoop/lib/*.jar,/usr/lib/hadoop-hdfs/*.jar,/usr/lib/hadoop-hdfs/lib/*.jar,/usr/lib/hadoop-mapreduce/*.jar,/usr/lib/hadoop-mapreduce/lib/*.jar,/usr/lib/hadoop-yarn/*.jar,/usr/lib/hadoop-yarn/lib/*.jar,/usr/lib/hive/lib/*.jar # # List of comma-separated paths defining the contents of the "server" http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3d7434e..e3431bd 100644 --- a/pom.xml +++ b/pom.xml @@ -97,6 +97,7 @@ limitations under the License. <commons-codec.version>1.9</commons-codec.version> <derby.version>10.8.2.2</derby.version> <hadoop.2.version>2.6.0</hadoop.2.version> + <hive.version>0.13.1</hive.version> <guava.version>11.0.2</guava.version> <json-simple.version>1.1</json-simple.version> <testng.version>6.8.13</testng.version> @@ -349,26 +350,35 @@ limitations under the License. <version>${hadoop.2.version}</version> <scope>provided</scope> </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-jobclient</artifactId> <version>${hadoop.2.version}</version> <scope>provided</scope> </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> <version>${hadoop.2.version}</version> </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> <version>${hadoop.2.version}</version> </dependency> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + <version>${hive.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-serde</artifactId> + <version>${hive.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.googlecode.json-simple</groupId> <artifactId>json-simple</artifactId> <version>${json-simple.version}</version> @@ -532,6 +542,11 @@ limitations under the License. <version>${kitesdk.version}</version> </dependency> <dependency> + <groupId>org.kitesdk</groupId> + <artifactId>kite-data-hive</artifactId> + <version>${kitesdk.version}</version> + </dependency> + <dependency> <groupId>org.apache.tomcat</groupId> <artifactId>catalina</artifactId> <version>${tomcat.version}</version>
