Repository: incubator-drill Updated Branches: refs/heads/master c6b181847 -> 83f3728e7
DRILL-1347: Update Hive storage plugin to Hive version 0.13.1 from current version 0.12.0. Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a36ce656 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a36ce656 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a36ce656 Branch: refs/heads/master Commit: a36ce656017609b5d7893c749d77495b7ed3a0ad Parents: c6b1818 Author: vkorukanti <venki.koruka...@gmail.com> Authored: Thu Aug 28 08:49:35 2014 -0700 Committer: vkorukanti <venki.koruka...@gmail.com> Committed: Mon Nov 3 10:31:08 2014 -0800 ---------------------------------------------------------------------- .../codegen/templates/ObjectInspectors.java | 7 ++- .../drill/exec/expr/fn/HiveFuncHolder.java | 2 +- .../exec/expr/fn/HiveFunctionRegistry.java | 2 +- .../AbstractDrillPrimitiveObjectInspector.java | 4 +- .../drill/exec/store/hive/HiveRecordReader.java | 5 +- .../exec/store/hive/schema/DrillHiveTable.java | 8 ++- .../exec/store/hive/HiveTestDataGenerator.java | 10 ++-- contrib/storage-hive/hive-exec-shade/pom.xml | 60 ++++++++++++++++++-- distribution/src/assemble/bin.xml | 9 +++ .../apache/drill/jdbc/test/TestMetadataDDL.java | 14 +++-- 10 files changed, 94 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java index 3345b75..022806a 100644 --- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java +++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.primitive.*; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -49,7 +50,7 @@ public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDri implements ${entry.hiveOI} { public Drill${entry.drillType}ObjectInspector() { - super(PrimitiveObjectInspectorUtils.${entry.hiveType?lower_case}TypeEntry); + super(TypeInfoFactory.${entry.hiveType?lower_case}TypeInfo); } <#if entry.drillType == "VarChar"> @@ -161,7 +162,7 @@ public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDri @Override public HiveDecimal getPrimitiveJavaObject(Object o){ Decimal38SparseHolder h = (Decimal38SparseHolder) o; - return new HiveDecimal(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale)); + return HiveDecimal.create(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale)); } } @@ -169,7 +170,7 @@ public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDri @Override public HiveDecimal getPrimitiveJavaObject(Object o){ NullableDecimal38SparseHolder h = (NullableDecimal38SparseHolder) o; - return new HiveDecimal(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale)); + return HiveDecimal.create(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale)); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java index 2861b97..1c0b02b 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java @@ -161,7 +161,7 @@ public class HiveFuncHolder extends AbstractFuncHolder { return JExpr._new(m.directClass(GenericUDFBridge.class.getCanonicalName())) .arg(JExpr.lit(udfName)) .arg(JExpr.lit(false)) - .arg(JExpr.dotclass(m.directClass(udfClazz.getCanonicalName()))); + .arg(JExpr.lit(udfClazz.getCanonicalName().toString())); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java index 5fba542..13fb107 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java @@ -186,7 +186,7 @@ public class HiveFunctionRegistry implements PluggableFunctionRegistry{ MajorType[] argTypes, ObjectInspector[] argOIs) { try { - GenericUDF udfInstance = new GenericUDFBridge(udfName, false/* is operator */, udfClazz); + GenericUDF udfInstance = new GenericUDFBridge(udfName, false/* is operator */, udfClazz.getName()); ObjectInspector returnOI = udfInstance.initialize(argOIs); return new HiveFuncHolder( http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java index 2a98398..b451086 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java @@ -18,12 +18,12 @@ package org.apache.drill.exec.expr.fn.impl.hive; import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; public abstract class AbstractDrillPrimitiveObjectInspector extends AbstractPrimitiveObjectInspector { - public AbstractDrillPrimitiveObjectInspector(PrimitiveTypeEntry typeEntry) { + public AbstractDrillPrimitiveObjectInspector(PrimitiveTypeInfo typeEntry) { super(typeEntry); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java index c5831a0..24b6924 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java @@ -189,8 +189,7 @@ public class HiveRecordReader extends AbstractRecordReader { selectedColumnNames.add(columnName); } } - ColumnProjectionUtils.appendReadColumnIDs(job, columnIds); - ColumnProjectionUtils.appendReadColumnNames(job, selectedColumnNames); + ColumnProjectionUtils.appendReadColumns(job, columnIds, selectedColumnNames); } for (String columnName : selectedColumnNames) { @@ -521,7 +520,7 @@ public class HiveRecordReader extends AbstractRecordReader { case BYTE: return Byte.parseByte(value); case DECIMAL: - return new HiveDecimal(value); + return HiveDecimal.create(value); case DOUBLE: return Double.parseDouble(value); case FLOAT: http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java index 41f6e53..a843146 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java @@ -26,6 +26,7 @@ import org.apache.drill.exec.store.hive.HiveReadEntry; import org.apache.drill.exec.store.hive.HiveStoragePlugin; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -107,9 +108,10 @@ public class DrillHiveTable extends DrillTable{ case BINARY: return typeFactory.createSqlType(SqlTypeName.BINARY); - case DECIMAL: - final int precision = 38; // Hive 0.12 has standard precision - return typeFactory.createSqlType(SqlTypeName.DECIMAL, precision); + case DECIMAL: { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo)pTypeInfo; + return typeFactory.createSqlType(SqlTypeName.DECIMAL, decimalTypeInfo.precision(), decimalTypeInfo.scale()); + } case STRING: case VARCHAR: { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index d339d28..2eb4234 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -79,13 +79,13 @@ public class HiveTestDataGenerator { cleanDir(DB_DIR); cleanDir(WH_DIR); - HiveConf conf = new HiveConf(); + HiveConf conf = new HiveConf(SessionState.class); conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s;create=true", DB_DIR)); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); conf.set("hive.metastore.warehouse.dir", WH_DIR); - SessionState ss = new SessionState(new HiveConf(SessionState.class)); + SessionState ss = new SessionState(conf); SessionState.start(ss); hiveDriver = new Driver(conf); @@ -120,7 +120,7 @@ public class HiveTestDataGenerator { " binary_field BINARY," + " boolean_field BOOLEAN," + " tinyint_field TINYINT," + - " decimal_field DECIMAL," + + " decimal_field DECIMAL(38, 3)," + " double_field DOUBLE," + " float_field FLOAT," + " int_field INT," + @@ -134,7 +134,7 @@ public class HiveTestDataGenerator { " binary_part BINARY," + " boolean_part BOOLEAN," + " tinyint_part TINYINT," + - " decimal_part DECIMAL," + + " decimal_part DECIMAL(38, 3)," + " double_part DOUBLE," + " float_part FLOAT," + " int_part INT," + @@ -194,7 +194,7 @@ public class HiveTestDataGenerator { "dataType DATE, " + "timestampType TIMESTAMP, " + "binaryType BINARY, " + - "decimalType DECIMAL, " + + "decimalType DECIMAL(38, 2), " + "stringType STRING, " + "varCharType VARCHAR(20), " + "listType ARRAY<STRING>, " + http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/contrib/storage-hive/hive-exec-shade/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml b/contrib/storage-hive/hive-exec-shade/pom.xml index e0cd509..7faa393 100644 --- a/contrib/storage-hive/hive-exec-shade/pom.xml +++ b/contrib/storage-hive/hive-exec-shade/pom.xml @@ -30,11 +30,15 @@ <packaging>jar</packaging> <name>contrib/hive-storage-plugin/hive-exec-shaded</name> + <properties> + <hive.version>0.13.1</hive.version> + </properties> + <dependencies> <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-exec</artifactId> - <version>0.12.0</version> + <version>${hive.version}</version> <scope>compile</scope> <exclusions> <exclusion> @@ -70,9 +74,13 @@ <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-metastore</artifactId> - <version>0.12.0</version> + <version>${hive.version}</version> <exclusions> <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-serde</artifactId> + </exclusion> + <exclusion> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> </exclusion> @@ -93,7 +101,7 @@ <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-hbase-handler</artifactId> - <version>0.12.0</version> + <version>${hive.version}</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> @@ -103,6 +111,34 @@ <groupId>org.apache.hbase</groupId> <artifactId>hbase</artifactId> </exclusion> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-service</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-exec</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-shims</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-serde</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-common</artifactId> + </exclusion> </exclusions> </dependency> </dependencies> @@ -123,6 +159,14 @@ <artifactSet> <includes> <include>org.apache.hive:hive-exec</include> + <include>com.twitter:parquet-column</include> + <include>com.twitter:parquet-hadoop</include> + <include>commons-codec:commons-codec</include> + <include>com.twitter:parquet-format</include> + <include>com.twitter:parquet-common</include> + <include>com.twitter:parquet-jackson</include> + <include>com.twitter:parquet-encoding</include> + <include>com.twitter:parquet-generator</include> </includes> </artifactSet> <createDependencyReducedPom>false</createDependencyReducedPom> @@ -130,7 +174,15 @@ <relocations> <relocation> <pattern>com.google.</pattern> - <shadedPattern>com.google.hive12.</shadedPattern> + <shadedPattern>hive.com.google.</shadedPattern> + </relocation> + <relocation> + <pattern>parquet.</pattern> + <shadedPattern>hive.parquet.</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons.codec.</pattern> + <shadedPattern>hive.org.apache.commons.codec.</shadedPattern> </relocation> </relocations> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/distribution/src/assemble/bin.xml ---------------------------------------------------------------------- diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml index 4fba7f8..e7c84dd 100644 --- a/distribution/src/assemble/bin.xml +++ b/distribution/src/assemble/bin.xml @@ -112,6 +112,15 @@ <exclude>org.apache.drill.exec</exclude> <exclude>org.apache.drill.contrib</exclude> <exclude>org.apache.drill.contrib.storage-hive</exclude> + <!-- Below Hive jars are already included in drill-hive-exec-shaded.jar --> + <exclude>org.apache.hive:hive-common</exclude> + <exclude>org.apache.hive:hive-serde</exclude> + <exclude>org.apache.hive:hive-shims</exclude> + <exclude>org.apache.hive.shims:hive-shims-0.20</exclude> + <exclude>org.apache.hive.shims:hive-shims-0.20S</exclude> + <exclude>org.apache.hive.shims:hive-shims-0.23</exclude> + <exclude>org.apache.hive.shims:hive-shims-common</exclude> + <exclude>org.apache.hive.shims:hive-shims-common-secure</exclude> <exclude>org.hsqldb:hsqldb</exclude> <exclude>junit:junit:jar</exclude> <exclude>com.googlecode.jmockit:jmockit:jar</exclude> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a36ce656/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java ---------------------------------------------------------------------- diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java index 948f600..c52eafd 100644 --- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java +++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java @@ -269,16 +269,20 @@ public class TestMetadataDDL extends JdbcTestQueryBase { @Test public void testVarCharMaxLengthAndDecimalPrecisionInInfoSchema() throws Exception{ JdbcAssert.withNoDefaultSchema() - .sql("SELECT COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION " + + .sql("SELECT COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE " + "FROM INFORMATION_SCHEMA.`COLUMNS` " + "WHERE TABLE_SCHEMA = 'hive_test.default' AND TABLE_NAME = 'infoschematest' AND " + "(COLUMN_NAME = 'stringtype' OR COLUMN_NAME = 'varchartype' OR " + "COLUMN_NAME = 'inttype' OR COLUMN_NAME = 'decimaltype')") .returnsSet(ImmutableSet.of( - "COLUMN_NAME=inttype; DATA_TYPE=INTEGER; CHARACTER_MAXIMUM_LENGTH=-1; NUMERIC_PRECISION=-1", - "COLUMN_NAME=decimaltype; DATA_TYPE=DECIMAL; CHARACTER_MAXIMUM_LENGTH=-1; NUMERIC_PRECISION=38", - "COLUMN_NAME=stringtype; DATA_TYPE=VARCHAR; CHARACTER_MAXIMUM_LENGTH=65535; NUMERIC_PRECISION=-1", - "COLUMN_NAME=varchartype; DATA_TYPE=VARCHAR; CHARACTER_MAXIMUM_LENGTH=20; NUMERIC_PRECISION=-1")); + "COLUMN_NAME=inttype; DATA_TYPE=INTEGER; CHARACTER_MAXIMUM_LENGTH=-1; " + + "NUMERIC_PRECISION=-1; NUMERIC_SCALE=-1", + "COLUMN_NAME=decimaltype; DATA_TYPE=DECIMAL; CHARACTER_MAXIMUM_LENGTH=-1; " + + "NUMERIC_PRECISION=38; NUMERIC_SCALE=2", + "COLUMN_NAME=stringtype; DATA_TYPE=VARCHAR; CHARACTER_MAXIMUM_LENGTH=65535; " + + "NUMERIC_PRECISION=-1; NUMERIC_SCALE=-1", + "COLUMN_NAME=varchartype; DATA_TYPE=VARCHAR; CHARACTER_MAXIMUM_LENGTH=20; " + + "NUMERIC_PRECISION=-1; NUMERIC_SCALE=-1")); } @Test
