Repository: drill Updated Branches: refs/heads/master 4e2df00e7 -> d32acdb90
DRILL-745: Add a test case for Hive avro tables. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b46c53eb Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b46c53eb Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b46c53eb Branch: refs/heads/master Commit: b46c53eb8e82d19fd5243ff2a9050ca8aae1346b Parents: ae62407 Author: vkorukanti <[email protected]> Authored: Fri Jun 12 14:18:08 2015 -0700 Committer: vkorukanti <[email protected]> Committed: Wed Jun 17 13:33:15 2015 -0700 ---------------------------------------------------------------------- contrib/storage-hive/core/pom.xml | 5 ++ .../apache/drill/exec/hive/TestHiveStorage.java | 10 ++++ .../exec/hive/TestInfoSchemaOnHiveStorage.java | 1 + .../exec/store/hive/HiveTestDataGenerator.java | 23 ++++++++ .../src/test/resources/avro_test_schema.json | 9 +++ pom.xml | 62 +++++++++++++++++--- 6 files changed, 102 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index 7bf644a..fb96a0d 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -78,6 +78,11 @@ <classifier>tests</classifier> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java index 27ba9fe..2898f91 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java @@ -176,4 +176,14 @@ public class TestHiveStorage extends HiveTestBase { .baselineValues(1, " key_1") .go(); } + + @Test // DRILL-745 + public void queryingHiveAvroTable() throws Exception { + testBuilder() + .sqlQuery("SELECT * FROM hive.db1.avro ORDER BY key DESC LIMIT 1") + .unOrdered() + .baselineColumns("key", "value") + .baselineValues(5, " key_5") + .go(); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java index 0f578ad..b684538 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java @@ -45,6 +45,7 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase { .unOrdered() .baselineColumns("TABLE_SCHEMA", "TABLE_NAME") .baselineValues("hive.db1", "kv_db1") + .baselineValues("hive.db1", "avro") .go(); } http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index d47d035..5b60c05 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -26,10 +26,12 @@ import java.util.Map; import com.google.common.io.Files; import org.apache.commons.io.FileUtils; +import org.apache.drill.BaseTestQuery; import org.apache.drill.common.exceptions.DrillException; import org.apache.drill.exec.store.StoragePluginRegistry; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; @@ -118,6 +120,8 @@ public class HiveTestDataGenerator { conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s;create=true", dbDir)); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); conf.set("hive.metastore.warehouse.dir", whDir); + conf.set("mapred.job.tracker", "local"); + conf.set("hive.exec.scratchdir", Files.createTempDir().getAbsolutePath() + File.separator + "scratch_dir"); SessionState ss = new SessionState(conf); SessionState.start(ss); @@ -130,6 +134,16 @@ public class HiveTestDataGenerator { executeQuery(hiveDriver, "CREATE DATABASE IF NOT EXISTS db1"); createTableAndLoadData(hiveDriver, "db1", "kv_db1", testDataFile); + // Create an Avro format based table backed by schema in a separate file + final String avroCreateQuery = String.format("CREATE TABLE db1.avro " + + "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' " + + "STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' " + + "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' " + + "TBLPROPERTIES ('avro.schema.url'='file:///%s')", getSchemaFile("avro_test_schema.json")); + + executeQuery(hiveDriver, avroCreateQuery); + executeQuery(hiveDriver, "INSERT INTO TABLE db1.avro SELECT * FROM default.kv"); + executeQuery(hiveDriver, "USE default"); // create a table with no data @@ -302,6 +316,15 @@ public class HiveTestDataGenerator { return file.getPath(); } + private String getSchemaFile(final String resource) throws Exception { + final File file = getTempFile(); + PrintWriter printWriter = new PrintWriter(file); + printWriter.write(BaseTestQuery.getFile(resource)); + printWriter.close(); + + return file.getPath(); + } + private String generateTestDataFileWithDate() throws Exception { final File file = getTempFile(); http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/resources/avro_test_schema.json ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/resources/avro_test_schema.json b/contrib/storage-hive/core/src/test/resources/avro_test_schema.json new file mode 100644 index 0000000..d91d0fd --- /dev/null +++ b/contrib/storage-hive/core/src/test/resources/avro_test_schema.json @@ -0,0 +1,9 @@ +{ + "namespace": "drill.test", + "name": "test_serializer", + "type": "record", + "fields": [ + { "name" : "key", "type" : "int" }, + { "name" : "value", "type" : "string" } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2d7a9cd..1b14ae1 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ Apache Hive 1.0.0. If the version is changed, make sure the jars and their dependencies are updated. --> <hive.version>1.0.0</hive.version> + <hadoop.version>2.4.1</hadoop.version> </properties> <scm> @@ -561,6 +562,26 @@ <dependencyManagement> <dependencies> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-exec</artifactId> <version>${hive.version}</version> @@ -755,7 +776,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>2.4.1</version> + <version>${hadoop.version}</version> <exclusions> <exclusion> <groupId>org.mortbay.jetty</groupId> @@ -846,7 +867,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>2.4.1</version> + <version>${hadoop.version}</version> <scope>test</scope> <classifier>tests</classifier> <exclusions> @@ -939,7 +960,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> - <version>2.4.1</version> + <version>${hadoop.version}</version> <exclusions> <exclusion> <groupId>javax.servlet</groupId> @@ -1219,7 +1240,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> - <version>2.4.1</version> + <version>${hadoop.version}</version> <scope>test</scope> <exclusions> <exclusion> @@ -1240,7 +1261,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> - <version>2.4.1</version> + <version>${hadoop.version}</version> <scope>test</scope> <classifier>tests</classifier> <exclusions> @@ -1271,13 +1292,14 @@ <alt-hadoop>mapr</alt-hadoop> <rat.excludeSubprojects>true</rat.excludeSubprojects> <hive.version>1.0.0-mapr-1504</hive.version> + <hadoop.version>2.5.1-mapr-1503</hadoop.version> </properties> <dependencyManagement> <dependencies> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>2.5.1-mapr-1503</version> + <version>${hadoop.version}</version> <exclusions> <exclusion> <artifactId>commons-logging</artifactId> @@ -1369,7 +1391,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> - <version>2.5.1-mapr-1503</version> + <version>${hadoop.version}</version> <exclusions> <exclusion> <artifactId>commons-logging</artifactId> @@ -1521,7 +1543,23 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> - <version>2.5.1-mapr-1503</version> + <version>${hadoop.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + <exclusion> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + <version>${hadoop.version}</version> <scope>test</scope> <exclusions> <exclusion> @@ -1529,9 +1567,17 @@ <artifactId>commons-logging</artifactId> </exclusion> <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>servlet-api</artifactId> + </exclusion> + <exclusion> <groupId>javax.servlet</groupId> <artifactId>servlet-api</artifactId> </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> </exclusions> </dependency> <dependency>
