Repository: sqoop Updated Branches: refs/heads/sqoop2 3ba34e250 -> a18d27e37
SQOOP-2238: Sqoop2: Parquet jars missing (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/a18d27e3 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/a18d27e3 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/a18d27e3 Branch: refs/heads/sqoop2 Commit: a18d27e3711cbf4a00c70492a3bbca19bf12e3c9 Parents: 3ba34e2 Author: Jarek Jarcec Cecho <[email protected]> Authored: Sun Mar 29 13:20:22 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Sun Mar 29 13:20:22 2015 -0700 ---------------------------------------------------------------------- .../apache/sqoop/connector/kite/KiteFromInitializer.java | 9 +++++++++ .../apache/sqoop/connector/kite/KiteToInitializer.java | 11 +++++++++++ 2 files changed, 20 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/a18d27e3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java index 06f18e7..fd3017b 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java @@ -59,6 +59,14 @@ public class KiteFromInitializer extends Initializer<LinkConfiguration, jars.add(ClassUtils.jarForClass("org.kitesdk.data.Datasets")); jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.databind.JsonNode")); jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.core.TreeNode")); + jars.add(ClassUtils.jarForClass("parquet.hadoop.metadata.CompressionCodecName")); + jars.add(ClassUtils.jarForClass("parquet.format.CompressionCodec")); + jars.add(ClassUtils.jarForClass("parquet.avro.AvroParquetWriter")); + jars.add(ClassUtils.jarForClass("parquet.column.ParquetProperties")); + jars.add(ClassUtils.jarForClass("parquet.Version")); + jars.add(ClassUtils.jarForClass("parquet.org.codehaus.jackson.type.TypeReference")); + jars.add(ClassUtils.jarForClass("parquet.bytes.CapacityByteArrayOutputStream")); + jars.add(ClassUtils.jarForClass("parquet.encoding.Generator")); if (fromJobConfig.fromJobConfig.uri.startsWith("dataset:hive")) { // @TODO(Abe): Remove a deps that aren't used? jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.conf.HiveConf")); @@ -72,6 +80,7 @@ public class KiteFromInitializer extends Initializer<LinkConfiguration, jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler")); jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery")); jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement")); + jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe")); } return jars; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/a18d27e3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java index 39a2989..e27c0b8 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java @@ -65,6 +65,16 @@ public class KiteToInitializer extends Initializer<LinkConfiguration, if (FileFormat.CSV.equals(toJobConfig.toJobConfig.fileFormat)) { jars.add(ClassUtils.jarForClass("au.com.bytecode.opencsv.CSVWriter")); } + if (FileFormat.PARQUET.equals(toJobConfig.toJobConfig.fileFormat)) { + jars.add(ClassUtils.jarForClass("parquet.hadoop.metadata.CompressionCodecName")); + jars.add(ClassUtils.jarForClass("parquet.format.CompressionCodec")); + jars.add(ClassUtils.jarForClass("parquet.avro.AvroParquetWriter")); + jars.add(ClassUtils.jarForClass("parquet.column.ParquetProperties")); + jars.add(ClassUtils.jarForClass("parquet.Version")); + jars.add(ClassUtils.jarForClass("parquet.org.codehaus.jackson.type.TypeReference")); + jars.add(ClassUtils.jarForClass("parquet.bytes.CapacityByteArrayOutputStream")); + jars.add(ClassUtils.jarForClass("parquet.encoding.Generator")); + } if (toJobConfig.toJobConfig.uri.startsWith("dataset:hive")) { // @TODO(Abe): Remove a deps that aren't used? jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.conf.HiveConf")); @@ -78,6 +88,7 @@ public class KiteToInitializer extends Initializer<LinkConfiguration, jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler")); jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery")); jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement")); + jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe")); } return jars; }
