DRILL-3266: Add hive-contrib dependency to Hive storage plugin Module hive-contrib has SerDes and UDF implementations which we want them to be available by default in Drill Hive storage plugin.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/d32acdb9 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/d32acdb9 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/d32acdb9 Branch: refs/heads/master Commit: d32acdb9020aaaba656588a03a02dd42e6a61241 Parents: b46c53e Author: vkorukanti <[email protected]> Authored: Mon Jun 15 08:11:10 2015 -0700 Committer: vkorukanti <[email protected]> Committed: Wed Jun 17 14:25:35 2015 -0700 ---------------------------------------------------------------------- contrib/storage-hive/core/pom.xml | 4 +++ .../apache/drill/exec/hive/TestHiveStorage.java | 10 ++++++++ .../exec/hive/TestInfoSchemaOnHiveStorage.java | 20 ++++++++++++--- .../exec/store/hive/HiveTestDataGenerator.java | 24 +++++++++-------- pom.xml | 27 ++++++++++++++++++++ 5 files changed, 71 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index fb96a0d..546fd7b 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -61,6 +61,10 @@ <artifactId>hive-hbase-handler</artifactId> </dependency> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-contrib</artifactId> + </dependency> + <dependency> <groupId>org.apache.calcite</groupId> <artifactId>calcite-core</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java index 2898f91..39e8f54 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java @@ -186,4 +186,14 @@ public class TestHiveStorage extends HiveTestBase { .baselineValues(5, " key_5") .go(); } + + @Test // DRILL-3266 + public void queryingTableWithSerDeInHiveContribJar() throws Exception { + testBuilder() + .sqlQuery("SELECT * FROM hive.db1.kv_db1 ORDER BY key DESC LIMIT 1") + .unOrdered() + .baselineColumns("key", "value") + .baselineValues("5", " key_5") + .go(); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java index b684538..6118be5 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java @@ -93,7 +93,13 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase { // When table name is fully qualified with schema name (sub-schema is non-default schema) @Test public void describeTable2() throws Exception{ - describeHelper(null, "DESCRIBE hive.`db1`.kv_db1"); + testBuilder() + .sqlQuery("DESCRIBE hive.`db1`.kv_db1") + .unOrdered() + .baselineColumns(baselineCols) + .baselineValues("key", "CHARACTER VARYING", "YES") + .baselineValues("value", "CHARACTER VARYING", "YES") + .go(); } // When table is qualified with just the top level schema. It should look for the table in default sub-schema within @@ -113,7 +119,13 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase { // given as single level schema name. @Test public void describeTable5() throws Exception { - describeHelper(null, "DESCRIBE `hive.db1`.kv_db1"); + testBuilder() + .sqlQuery("DESCRIBE `hive.db1`.kv_db1") + .unOrdered() + .baselineColumns(baselineCols) + .baselineValues("key", "CHARACTER VARYING", "YES") + .baselineValues("value", "CHARACTER VARYING", "YES") + .go(); } // When current default schema is just the top-level schema name and the table has no schema qualifier. It should @@ -187,8 +199,8 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase { .unOrdered() .optionSettingQueriesForTestQuery("USE hive.db1") .baselineColumns("key", "value") - .baselineValues(1, " key_1") - .baselineValues(2, " key_2") + .baselineValues("1", " key_1") + .baselineValues("2", " key_2") .go(); } } http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index 5b60c05..ea8d90f 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -130,9 +130,21 @@ public class HiveTestDataGenerator { // generate (key, value) test data String testDataFile = generateTestDataFile(); - createTableAndLoadData(hiveDriver, "default", "kv", testDataFile); + // Create a (key, value) schema table with Text SerDe which is available in hive-serdes.jar + executeQuery(hiveDriver, "CREATE TABLE IF NOT EXISTS default.kv(key INT, value STRING) " + + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE"); + executeQuery(hiveDriver, "LOAD DATA LOCAL INPATH '" + testDataFile + "' OVERWRITE INTO TABLE default.kv"); + + // Create a (key, value) schema table in non-default database with RegexSerDe which is available in hive-contrib.jar + // Table with RegExSerde is expected to have columns of STRING type only. executeQuery(hiveDriver, "CREATE DATABASE IF NOT EXISTS db1"); - createTableAndLoadData(hiveDriver, "db1", "kv_db1", testDataFile); + executeQuery(hiveDriver, "CREATE TABLE db1.kv_db1(key STRING, value STRING) " + + "ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.RegexSerDe' " + + "WITH SERDEPROPERTIES (" + + " \"input.regex\" = \"([0-9]*), (.*_[0-9]*)\", " + + " \"output.format.string\" = \"%1$s, %2$s\"" + + ") "); + executeQuery(hiveDriver, "INSERT INTO TABLE db1.kv_db1 SELECT * FROM default.kv"); // Create an Avro format based table backed by schema in a separate file final String avroCreateQuery = String.format("CREATE TABLE db1.avro " + @@ -293,14 +305,6 @@ public class HiveTestDataGenerator { ss.close(); } - private void createTableAndLoadData(Driver hiveDriver, String dbName, String tblName, String dataFile) { - executeQuery(hiveDriver, String.format("USE %s", dbName)); - executeQuery(hiveDriver, String.format("CREATE TABLE IF NOT EXISTS %s.%s(key INT, value STRING) "+ - "ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE", dbName, tblName)); - executeQuery(hiveDriver, - String.format("LOAD DATA LOCAL INPATH '%s' OVERWRITE INTO TABLE %s.%s", dataFile, dbName, tblName)); - } - private File getTempFile() throws Exception { return java.nio.file.Files.createTempFile("drill-hive-test", ".txt").toFile(); } http://git-wip-us.apache.org/repos/asf/drill/blob/d32acdb9/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 1b14ae1..675a6e0 100644 --- a/pom.xml +++ b/pom.xml @@ -711,6 +711,33 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-contrib</artifactId> + <version>${hive.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-serde</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-shims</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + <exclusion> + <artifactId>slf4j-log4j12</artifactId> + <groupId>org.slf4j</groupId> + </exclusion> + </exclusions> + </dependency> </dependencies> </dependencyManagement>
