Repository: drill Updated Branches: refs/heads/master 45a82c457 -> d5dc322d5
DRILL-3273: Pass an empty DeferredObject to Hive UDFs for null argument value + Handle nulls in ObjectInspector implementations for Drill types. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/50116591 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/50116591 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/50116591 Branch: refs/heads/master Commit: 501165914458942f8cdb3f80bd9f953facd2dcb8 Parents: 45a82c4 Author: vkorukanti <[email protected]> Authored: Wed Jun 10 15:54:06 2015 -0700 Committer: vkorukanti <[email protected]> Committed: Fri Jun 12 14:23:32 2015 -0700 ---------------------------------------------------------------------- .../codegen/templates/ObjectInspectors.java | 319 +++++++++++-------- .../drill/exec/expr/fn/HiveFuncHolder.java | 4 +- .../drill/exec/fn/hive/HiveTestUDFImpls.java | 2 +- .../drill/exec/fn/hive/TestInbuiltHiveUDFs.java | 46 +++ .../drill/exec/fn/hive/TestSampleHiveUDFs.java | 4 +- .../apache/drill/exec/hive/TestHiveStorage.java | 6 +- .../exec/store/hive/HiveTestDataGenerator.java | 3 +- 7 files changed, 243 insertions(+), 141 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/50116591/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java index 022806a..10379ff 100644 --- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java +++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java @@ -46,205 +46,264 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; -public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDrillPrimitiveObjectInspector - implements ${entry.hiveOI} { +public class Drill${entry.drillType}ObjectInspector { +<#assign seq = ["Required", "Optional"]> +<#list seq as mode> - public Drill${entry.drillType}ObjectInspector() { - super(TypeInfoFactory.${entry.hiveType?lower_case}TypeInfo); - } + public static class ${mode} extends AbstractDrillPrimitiveObjectInspector implements ${entry.hiveOI} { + public ${mode}() { + super(TypeInfoFactory.${entry.hiveType?lower_case}TypeInfo); + } <#if entry.drillType == "VarChar"> - @Override - public HiveVarcharWritable getPrimitiveWritableObject(Object o) { - HiveVarcharWritable valW = new HiveVarcharWritable(); - valW.set(getPrimitiveJavaObject(o)); - return valW; - } - - public static class Required extends Drill${entry.drillType}ObjectInspector { @Override - public HiveVarchar getPrimitiveJavaObject(Object o) { - VarCharHolder h = (VarCharHolder)o; - String s = StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer); - return new HiveVarchar(s, HiveVarchar.MAX_VARCHAR_LENGTH); + public HiveVarcharWritable getPrimitiveWritableObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableVarCharHolder h = (NullableVarCharHolder)o; + <#else> + final VarCharHolder h = (VarCharHolder)o; + </#if> + final HiveVarcharWritable valW = new HiveVarcharWritable(); + valW.set(StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer), HiveVarchar.MAX_VARCHAR_LENGTH); + return valW; } - } - public static class Optional extends Drill${entry.drillType}ObjectInspector { @Override public HiveVarchar getPrimitiveJavaObject(Object o) { - NullableVarCharHolder h = (NullableVarCharHolder)o; - String s = h.isSet == 0 ? null : StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer); + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableVarCharHolder h = (NullableVarCharHolder)o; + <#else> + final VarCharHolder h = (VarCharHolder)o; + </#if> + final String s = StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer); return new HiveVarchar(s, HiveVarchar.MAX_VARCHAR_LENGTH); } - } - <#elseif entry.drillType == "Var16Char"> -@Override - public Text getPrimitiveWritableObject(Object o) { - throw new UnsupportedOperationException(); - } - - public static class Required extends Drill${entry.drillType}ObjectInspector { @Override - public String getPrimitiveJavaObject(Object o){ - Var16CharHolder h = (Var16CharHolder)o; - String s = StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer); - return s; + public Text getPrimitiveWritableObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableVar16CharHolder h = (NullableVar16CharHolder)o; + <#else> + final Var16CharHolder h = (Var16CharHolder)o; + </#if> + return new Text(StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer)); } - } - public static class Optional extends Drill${entry.drillType}ObjectInspector { @Override public String getPrimitiveJavaObject(Object o){ - NullableVar16CharHolder h = (NullableVar16CharHolder)o; - String s = h.isSet == 0 ? null : StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer); - return s; + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableVar16CharHolder h = (NullableVar16CharHolder)o; + <#else> + final Var16CharHolder h = (Var16CharHolder)o; + </#if> + return StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer); } - } <#elseif entry.drillType == "VarBinary"> - @Override - public BytesWritable getPrimitiveWritableObject(Object o) { - return new BytesWritable(getPrimitiveJavaObject(o)); - } - - public static class Required extends Drill${entry.drillType}ObjectInspector { @Override - public byte[] getPrimitiveJavaObject(Object o) { - VarBinaryHolder h = (VarBinaryHolder)o; - byte[] buf = new byte[h.end-h.start]; + public BytesWritable getPrimitiveWritableObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableVarBinaryHolder h = (NullableVarBinaryHolder)o; + <#else> + final VarBinaryHolder h = (VarBinaryHolder)o; + </#if> + final byte[] buf = new byte[h.end-h.start]; h.buffer.getBytes(h.start, buf, 0, h.end-h.start); - return buf; + return new BytesWritable(buf); } - } - public static class Optional extends Drill${entry.drillType}ObjectInspector { @Override public byte[] getPrimitiveJavaObject(Object o) { - NullableVarBinaryHolder h = (NullableVarBinaryHolder)o; - byte[] buf = new byte[h.end-h.start]; + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableVarBinaryHolder h = (NullableVarBinaryHolder)o; + <#else> + final VarBinaryHolder h = (VarBinaryHolder)o; + </#if> + final byte[] buf = new byte[h.end-h.start]; h.buffer.getBytes(h.start, buf, 0, h.end-h.start); return buf; } - } - <#elseif entry.drillType == "Bit"> - public static class Required extends Drill${entry.drillType}ObjectInspector { @Override public boolean get(Object o) { + <#if mode == "Optional"> + return ((NullableBitHolder)o).value == 0 ? false : true; + <#else> return ((BitHolder)o).value == 0 ? false : true; + </#if> } - } - public static class Optional extends Drill${entry.drillType}ObjectInspector { @Override - public boolean get(Object o) { - return ((NullableBitHolder)o).value == 0 ? false : true; + public BooleanWritable getPrimitiveWritableObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + return new BooleanWritable(((NullableBitHolder)o).value == 0 ? false : true); + <#else> + return new BooleanWritable(((BitHolder)o).value == 0 ? false : true); + </#if> } - } - - @Override - public BooleanWritable getPrimitiveWritableObject(Object o) { - return new BooleanWritable(get(o)); - } - - @Override - public Boolean getPrimitiveJavaObject(Object o) { - return new Boolean(get(o)); - } + @Override + public Boolean getPrimitiveJavaObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + return new Boolean(((NullableBitHolder)o).value == 0 ? false : true); + <#else> + return new Boolean(((BitHolder)o).value == 0 ? false : true); + </#if> + } <#elseif entry.drillType == "Decimal38Sparse"> - public HiveDecimalWritable getPrimitiveWritableObject(Object o) { - return new HiveDecimalWritable(getPrimitiveJavaObject(o)); - } - - public static class Required extends Drill${entry.drillType}ObjectInspector{ @Override public HiveDecimal getPrimitiveJavaObject(Object o){ - Decimal38SparseHolder h = (Decimal38SparseHolder) o; + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableDecimal38SparseHolder h = (NullableDecimal38SparseHolder) o; + <#else> + final Decimal38SparseHolder h = (Decimal38SparseHolder) o; + </#if> return HiveDecimal.create(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale)); } - } - public static class Optional extends Drill${entry.drillType}ObjectInspector{ @Override - public HiveDecimal getPrimitiveJavaObject(Object o){ - NullableDecimal38SparseHolder h = (NullableDecimal38SparseHolder) o; - return HiveDecimal.create(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale)); + public HiveDecimalWritable getPrimitiveWritableObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableDecimal38SparseHolder h = (NullableDecimal38SparseHolder) o; + <#else> + final Decimal38SparseHolder h = (Decimal38SparseHolder) o; + </#if> + return new HiveDecimalWritable( + HiveDecimal.create(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale))); } - } <#elseif entry.drillType == "TimeStamp"> - @Override - public TimestampWritable getPrimitiveWritableObject(Object o) { - return new TimestampWritable(getPrimitiveJavaObject(o)); - } - - public static class Required extends Drill${entry.drillType}ObjectInspector{ @Override public java.sql.Timestamp getPrimitiveJavaObject(Object o){ - return new java.sql.Timestamp(((TimeStampHolder)o).value); + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableTimeStampHolder h = (NullableTimeStampHolder) o; + <#else> + final TimeStampHolder h = (TimeStampHolder) o; + </#if> + return new java.sql.Timestamp(h.value); } - } - public static class Optional extends Drill${entry.drillType}ObjectInspector{ @Override - public java.sql.Timestamp getPrimitiveJavaObject(Object o){ - return new java.sql.Timestamp(((NullableTimeStampHolder)o).value); + public TimestampWritable getPrimitiveWritableObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableTimeStampHolder h = (NullableTimeStampHolder) o; + <#else> + final TimeStampHolder h = (TimeStampHolder) o; + </#if> + return new TimestampWritable(new java.sql.Timestamp(h.value)); } - } <#elseif entry.drillType == "Date"> - @Override - public DateWritable getPrimitiveWritableObject(Object o) { - return new DateWritable(getPrimitiveJavaObject(o)); - } - - public static class Required extends Drill${entry.drillType}ObjectInspector{ @Override public java.sql.Date getPrimitiveJavaObject(Object o){ - return new java.sql.Date(((DateHolder)o).value); + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableDateHolder h = (NullableDateHolder) o; + <#else> + final DateHolder h = (DateHolder) o; + </#if> + return new java.sql.Date(h.value); } - } - public static class Optional extends Drill${entry.drillType}ObjectInspector{ @Override - public java.sql.Date getPrimitiveJavaObject(Object o){ - return new java.sql.Date(((NullableDateHolder)o).value); + public DateWritable getPrimitiveWritableObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + final NullableDateHolder h = (NullableDateHolder) o; + <#else> + final DateHolder h = (DateHolder) o; + </#if> + return new DateWritable(new java.sql.Date(h.value)); } - } + <#else> + @Override + public ${entry.javaType} get(Object o){ + <#if mode == "Optional"> + return ((Nullable${entry.drillType}Holder)o).value; + <#else> + return ((${entry.drillType}Holder)o).value; + </#if> + } + <#if entry.drillType == "Int"> - @Override - public Integer getPrimitiveJavaObject(Object o) { - return new Integer(get(o)); - } + @Override + public Integer getPrimitiveJavaObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + </#if> + return new Integer(get(o)); + } <#else> - @Override - public ${entry.javaType?cap_first} getPrimitiveJavaObject(Object o) { - return new ${entry.javaType?cap_first}(get(o)); - } -</#if> - - @Override - public ${entry.javaType?cap_first}Writable getPrimitiveWritableObject(Object o) { - return new ${entry.javaType?cap_first}Writable(get(o)); - } - - public static class Required extends Drill${entry.drillType}ObjectInspector{ @Override - public ${entry.javaType} get(Object o){ - return((${entry.drillType}Holder)o).value; + public ${entry.javaType?cap_first} getPrimitiveJavaObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + return new ${entry.javaType?cap_first}(((Nullable${entry.drillType}Holder)o).value); + <#else> + return new ${entry.javaType?cap_first}(((${entry.drillType}Holder)o).value); + </#if> } - } +</#if> - public static class Optional extends Drill${entry.drillType}ObjectInspector{ @Override - public ${entry.javaType} get(Object o){ - return((Nullable${entry.drillType}Holder)o).value; + public ${entry.javaType?cap_first}Writable getPrimitiveWritableObject(Object o) { + <#if mode == "Optional"> + if (o == null) { + return null; + } + final Nullable${entry.drillType}Holder h = (Nullable${entry.drillType}Holder) o; + <#else> + final ${entry.drillType}Holder h = (${entry.drillType}Holder)o; + </#if> + return new ${entry.javaType?cap_first}Writable(h.value); } - } </#if> + } +</#list> } </#list> http://git-wip-us.apache.org/repos/asf/drill/blob/50116591/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java index 1c0b02b..a6e76b0 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java @@ -242,11 +242,11 @@ public class HiveFuncHolder extends AbstractFuncHolder { // initialize DeferredObject's. For an optional type, assign the value holder only if it is not null for(int i=0; i<argTypes.length; i++) { if (inputVariables[i].isOptional()) { + sub.assign(workspaceJVars[3].component(JExpr.lit(i)), workspaceJVars[2].component(JExpr.lit(i))); JBlock conditionalBlock = new JBlock(false, false); JConditional jc = conditionalBlock._if(inputVariables[i].getIsSet().ne(JExpr.lit(0))); - jc._then().assign(workspaceJVars[3].component(JExpr.lit(i)), workspaceJVars[2].component(JExpr.lit(i))); jc._then().assign(JExpr.ref(workspaceJVars[3].component(JExpr.lit(i)), "valueHolder"), inputVariables[i].getHolder()); - jc._else().assign(workspaceJVars[3].component(JExpr.lit(i)), JExpr._null()); + jc._else().assign(JExpr.ref(workspaceJVars[3].component(JExpr.lit(i)), "valueHolder"), JExpr._null()); sub.add(conditionalBlock); } else { sub.assign(workspaceJVars[3].component(JExpr.lit(i)), workspaceJVars[2].component(JExpr.lit(i))); http://git-wip-us.apache.org/repos/asf/drill/blob/50116591/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java index 8fc059b..31e4715 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java @@ -103,7 +103,7 @@ public class HiveTestUDFImpls { @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { - if (arguments[0] == null) { + if (arguments[0] == null || arguments[0].get() == null) { return null; } http://git-wip-us.apache.org/repos/asf/drill/blob/50116591/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java new file mode 100644 index 0000000..3ee832c --- /dev/null +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestInbuiltHiveUDFs.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.fn.hive; + +import org.apache.drill.exec.hive.HiveTestBase; +import org.junit.Test; + +public class TestInbuiltHiveUDFs extends HiveTestBase { + + @Test // DRILL-3273 + public void testConcatWS() throws Exception { + testBuilder() + .sqlQuery("SELECT concat_ws(string_field, string_part, '|') as rst from hive.readtest") + .unOrdered() + .baselineColumns("rst") + .baselineValues("stringstringfield|") + .baselineValues(new Object[] { null }) + .go(); + } + + @Test // DRILL-3273 + public void testEncode() throws Exception { + testBuilder() + .sqlQuery("SELECT encode(varchar_field, 'UTF-8') as rst from hive.readtest") + .unOrdered() + .baselineColumns("rst") + .baselineValues("varcharfield") + .baselineValues(new Object[] { null }) + .go(); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/50116591/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java index ac11752..86a78e5 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java @@ -91,14 +91,14 @@ public class TestSampleHiveUDFs extends HiveTestBase { @Test public void stringInOut() throws Exception{ String query = "SELECT testHiveUDFString(string_field) as col1 FROM hive.readtest"; - String expected = "col1\n" + "stringfield\n" + "\n"; + String expected = "col1\n" + "stringfield\n" + "null\n"; helper(query, expected); } @Test public void binaryInOut() throws Exception{ String query = "SELECT testHiveUDFBinary(binary_field) as col1 FROM hive.readtest"; - String expected = "col1\n" + "binaryfield\n" + "\n"; + String expected = "col1\n" + "binaryfield\n" + "null\n"; helper(query, expected); helper(query, expected); } http://git-wip-us.apache.org/repos/asf/drill/blob/50116591/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java index a25a524..27ba9fe 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java @@ -125,11 +125,7 @@ public class TestHiveStorage extends HiveTestBase { new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()), new DateTime(Date.valueOf("2013-07-05").getTime())) .baselineValues( // All fields are null, but partition fields have non-null values - "", // For binary (varchar, string too) empty value is considered as empty string instead of "null" - null, null, null, null, null, null, null, null, null, null, null, null, - "", // string_field - "", // varchar_field - null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "binary", true, (byte) 64, http://git-wip-us.apache.org/repos/asf/drill/blob/50116591/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index 5b867a7..d47d035 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -179,7 +179,8 @@ public class HiveTestDataGenerator { " varchar_part VARCHAR(50)," + " timestamp_part TIMESTAMP," + " date_part DATE" + - ") ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE" + ") ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' " + + "TBLPROPERTIES ('serialization.null.format'='') " ); // Add a partition to table 'readtest'
