Fix issues with Hive function generation to support DrillBuf
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/91870359 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/91870359 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/91870359 Branch: refs/heads/master Commit: 91870359947b8457f4613953fe59948fa9f2fe17 Parents: b5bbed8 Author: Jacques Nadeau <jacq...@apache.org> Authored: Mon Aug 25 21:49:46 2014 -0700 Committer: Jacques Nadeau <jacq...@apache.org> Committed: Mon Aug 25 21:49:46 2014 -0700 ---------------------------------------------------------------------- .../codegen/templates/ObjectInspectorHelper.java | 17 ++++++++++++----- .../main/codegen/templates/ObjectInspectors.java | 17 +++++++++++++---- .../apache/drill/exec/expr/fn/HiveFuncHolder.java | 2 +- 3 files changed, 26 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/91870359/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java index 091f027..961dfbb 100644 --- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java +++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java @@ -28,6 +28,7 @@ import com.sun.codemodel.*; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.expr.ClassGenerator; import org.apache.drill.exec.expr.DirectExpression; import org.apache.drill.exec.expr.TypeHelper; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -73,7 +74,7 @@ public class ObjectInspectorHelper { String.format("Type %s[%s] not supported as arguement to Hive UDFs", minorType.toString(), mode.toString())); } - public static JBlock initReturnValueHolder(JCodeModel m, JVar returnValueHolder, ObjectInspector oi, MinorType returnType) { + public static JBlock initReturnValueHolder(ClassGenerator<?> g, JCodeModel m, JVar returnValueHolder, ObjectInspector oi, MinorType returnType) { JBlock block = new JBlock(false, false); switch(oi.getCategory()) { case PRIMITIVE: { @@ -85,10 +86,16 @@ public class ObjectInspectorHelper { block.assign(returnValueHolder, JExpr._new(holderClass)); <#if entry.hiveType == "VARCHAR" || entry.hiveType == "STRING" || entry.hiveType == "BINARY"> - block.assign(returnValueHolder.ref("buffer"), - m.directClass(io.netty.buffer.Unpooled.class.getCanonicalName()) - .staticInvoke("wrappedBuffer") - .arg(JExpr.newArray(m._ref(byte.class), JExpr.lit(1000)))); + block.assign( // + returnValueHolder.ref("buffer"), // + g + .getMappingSet() + .getIncoming() + .invoke("getContext") + .invoke("getManagedBuffer") + .invoke("reallocIfNeeded") + .arg(JExpr.lit(1024)) + ); </#if> return block; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/91870359/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java index 576fa47..3345b75 100644 --- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java +++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java @@ -25,6 +25,7 @@ package org.apache.drill.exec.expr.fn.impl.hive; import org.apache.drill.exec.util.DecimalUtility; +import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers; import org.apache.drill.exec.expr.holders.*; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; @@ -62,14 +63,18 @@ public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDri public static class Required extends Drill${entry.drillType}ObjectInspector { @Override public HiveVarchar getPrimitiveJavaObject(Object o) { - return new HiveVarchar(((VarCharHolder)o).toString(), HiveVarchar.MAX_VARCHAR_LENGTH); + VarCharHolder h = (VarCharHolder)o; + String s = StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer); + return new HiveVarchar(s, HiveVarchar.MAX_VARCHAR_LENGTH); } } public static class Optional extends Drill${entry.drillType}ObjectInspector { @Override public HiveVarchar getPrimitiveJavaObject(Object o) { - return new HiveVarchar(((NullableVarCharHolder)o).toString(), HiveVarchar.MAX_VARCHAR_LENGTH); + NullableVarCharHolder h = (NullableVarCharHolder)o; + String s = h.isSet == 0 ? null : StringFunctionHelpers.toStringFromUTF8(h.start, h.end, h.buffer); + return new HiveVarchar(s, HiveVarchar.MAX_VARCHAR_LENGTH); } } @@ -82,14 +87,18 @@ public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDri public static class Required extends Drill${entry.drillType}ObjectInspector { @Override public String getPrimitiveJavaObject(Object o){ - return((Var16CharHolder)o).toString(); + Var16CharHolder h = (Var16CharHolder)o; + String s = StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer); + return s; } } public static class Optional extends Drill${entry.drillType}ObjectInspector { @Override public String getPrimitiveJavaObject(Object o){ - return((NullableVar16CharHolder)o).toString(); + NullableVar16CharHolder h = (NullableVar16CharHolder)o; + String s = h.isSet == 0 ? null : StringFunctionHelpers.toStringFromUTF16(h.start, h.end, h.buffer); + return s; } } <#elseif entry.drillType == "VarBinary"> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/91870359/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java index 45dbafe..2bc3576 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java @@ -218,7 +218,7 @@ public class HiveFuncHolder extends AbstractFuncHolder { ._throw(JExpr._new(m.directClass(RuntimeException.class.getCanonicalName())) .arg(JExpr.lit(String.format("Failed to initialize GenericUDF"))).arg(exVar)); - sub.add(ObjectInspectorHelper.initReturnValueHolder(m, workspaceJVars[4], returnOI, returnType.getMinorType())); + sub.add(ObjectInspectorHelper.initReturnValueHolder(g, m, workspaceJVars[4], returnOI, returnType.getMinorType())); // now add it to the doSetup block in Generated class JBlock setup = g.getBlock(ClassGenerator.BlockType.SETUP);
