Author: jvs
Date: Tue Oct 11 03:36:50 2011
New Revision: 1181615
URL: http://svn.apache.org/viewvc?rev=1181615&view=rev
Log:
HIVE-2470. Improve support for Constant Object Inspectors
(Jonathan Chang via jvs)
Added:
hive/trunk/ql/src/test/queries/clientpositive/constant_prop.q
hive/trunk/ql/src/test/results/clientpositive/constant_prop.q.out
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantListObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantMapObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java
Removed:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantVoidObjectInspector.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
Tue Oct 11 03:36:50 2011
@@ -26,7 +26,9 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
/**
* ExprNodeGenericFuncEvaluator.
@@ -41,6 +43,7 @@ public class ExprNodeGenericFuncEvaluato
transient GenericUDF genericUDF;
transient Object rowObject;
+ transient ObjectInspector outputOI;
transient ExprNodeEvaluator[] children;
transient GenericUDF.DeferredObject[] deferredChildren;
transient boolean isEager;
@@ -130,12 +133,17 @@ public class ExprNodeGenericFuncEvaluato
throw new HiveException(
"Stateful expressions cannot be used inside of CASE");
}
- return genericUDF.initialize(childrenOIs);
+ this.outputOI = genericUDF.initializeAndFoldConstants(childrenOIs);
+ return this.outputOI;
}
@Override
public Object evaluate(Object row) throws HiveException {
rowObject = row;
+ if (ObjectInspectorUtils.isConstantObjectInspector(outputOI)) {
+ // The output of this UDF is constant, so don't even bother evaluating.
+ return ((ConstantObjectInspector)outputOI).getWritableConstantValue();
+ }
if (isEager) {
for (int i = 0; i < deferredChildren.length; i++) {
((EagerExprObject) deferredChildren[i]).evaluate();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java
Tue Oct 11 03:36:50 2011
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -60,16 +61,34 @@ public class ExprNodeGenericFuncDesc ext
*/
private GenericUDF genericUDF;
private List<ExprNodeDesc> childExprs;
+ /**
+ * This class uses a writableObjectInspector rather than a TypeInfo to store
+ * the canonical type information for this NodeDesc.
+ */
+ private ObjectInspector writableObjectInspector;
public ExprNodeGenericFuncDesc() {
}
public ExprNodeGenericFuncDesc(TypeInfo typeInfo, GenericUDF genericUDF,
List<ExprNodeDesc> children) {
- super(typeInfo);
+
this(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo),
+ genericUDF, children);
+ }
+
+ public ExprNodeGenericFuncDesc(ObjectInspector oi, GenericUDF genericUDF,
+ List<ExprNodeDesc> children) {
+ super(TypeInfoUtils.getTypeInfoFromObjectInspector(oi));
+ this.writableObjectInspector =
+ ObjectInspectorUtils.getWritableObjectInspector(oi);
assert (genericUDF != null);
this.genericUDF = genericUDF;
- childExprs = children;
+ this.childExprs = children;
+ }
+
+ @Override
+ public ObjectInspector getWritableObjectInspector() {
+ return writableObjectInspector;
}
public GenericUDF getGenericUDF() {
@@ -190,9 +209,8 @@ public class ExprNodeGenericFuncDesc ext
}
}
- ObjectInspector oi = genericUDF.initialize(childrenOIs);
- return new ExprNodeGenericFuncDesc(TypeInfoUtils
- .getTypeInfoFromObjectInspector(oi), genericUDF, children);
+ ObjectInspector oi = genericUDF.initializeAndFoldConstants(childrenOIs);
+ return new ExprNodeGenericFuncDesc(oi, genericUDF, children);
}
@Override
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
Tue Oct 11 03:36:50 2011
@@ -18,16 +18,19 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
/**
* A Generic User-defined function (GenericUDF) for the use with Hive.
- *
+ *
* New GenericUDF classes need to inherit from this GenericUDF class.
- *
+ *
* The GenericUDF are superior to normal UDFs in the following ways: 1. It can
* accept arguments of complex types, and return complex types. 2. It can
accept
* variable length of arguments. 3. It can accept an infinite number of
function
@@ -47,6 +50,23 @@ public abstract class GenericUDF {
};
/**
+ * A basic dummy implementation of DeferredObject which just stores a Java
+ * Object reference.
+ */
+ public static class DeferredJavaObject implements DeferredObject {
+ private Object value;
+
+ public DeferredJavaObject(Object value) {
+ this.value = value;
+ }
+
+ @Override
+ public Object get() throws HiveException {
+ return value;
+ }
+ }
+
+ /**
* The constructor.
*/
public GenericUDF() {
@@ -66,8 +86,49 @@ public abstract class GenericUDF {
throws UDFArgumentException;
/**
+ * Initialize this GenericUDF. Additionally, if the arguments are constant
+ * and the function is eligible to be folded, then the constant value
+ * returned by this UDF will be computed and stored in the
+ * ConstantObjectInspector returned. Otherwise, the function behaves exactly
+ * like initialize().
+ */
+ public ObjectInspector initializeAndFoldConstants(ObjectInspector[]
arguments)
+ throws UDFArgumentException {
+
+ ObjectInspector oi = initialize(arguments);
+
+ boolean allConstant = true;
+ for (int ii = 0; ii < arguments.length; ++ii) {
+ if (!ObjectInspectorUtils.isConstantObjectInspector(arguments[ii])) {
+ allConstant = false;
+ break;
+ }
+ }
+
+ if (allConstant &&
+ !ObjectInspectorUtils.isConstantObjectInspector(oi) &&
+ FunctionRegistry.isDeterministic(this) &&
+ !FunctionRegistry.isStateful(this) &&
+ ObjectInspectorUtils.supportsConstantObjectInspector(oi)) {
+ DeferredObject[] argumentValues =
+ new DeferredJavaObject[arguments.length];
+ for (int ii = 0; ii < arguments.length; ++ii) {
+ argumentValues[ii] = new DeferredJavaObject(
+
((ConstantObjectInspector)arguments[ii]).getWritableConstantValue());
+ }
+ try {
+ Object constantValue = evaluate(argumentValues);
+ oi = ObjectInspectorUtils.getConstantObjectInspector(oi,
constantValue);
+ } catch (HiveException e) {
+ throw new UDFArgumentException(e);
+ }
+ }
+ return oi;
+ }
+
+ /**
* Evaluate the GenericUDF with the arguments.
- *
+ *
* @param arguments
* The arguments as DeferedObject, use DeferedObject.get() to get
the
* actual argument Object. The Objects can be inspected by the
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java
Tue Oct 11 03:36:50 2011
@@ -67,7 +67,7 @@ public ObjectInspector initialize(Object
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
- assert (arguments.length == 2);
+ assert (arguments.length == 1);
Object b = arguments[0].get();
ListObjectInspector lloi = (ListObjectInspector) bitmapOI;
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
Tue Oct 11 03:36:50 2011
@@ -97,8 +97,11 @@ public final class GenericUDFUtils {
}
if (returnObjectInspector == null) {
- // The first argument, just set it.
- returnObjectInspector = oi;
+ // The first argument, just set the return to be the standard
+ // writable version of this OI.
+ returnObjectInspector = ObjectInspectorUtils
+ .getStandardObjectInspector(oi,
+ ObjectInspectorCopyOption.WRITABLE);
return true;
}
Added: hive/trunk/ql/src/test/queries/clientpositive/constant_prop.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/constant_prop.q?rev=1181615&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/constant_prop.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/constant_prop.q Tue Oct 11
03:36:50 2011
@@ -0,0 +1,20 @@
+EXPLAIN
+SELECT NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ),
+ NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ).F2
+ FROM src LIMIT 1;
+
+SELECT NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ),
+ NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ).F2
+ FROM src LIMIT 1;
Added: hive/trunk/ql/src/test/results/clientpositive/constant_prop.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/constant_prop.q.out?rev=1181615&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/constant_prop.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/constant_prop.q.out Tue Oct
11 03:36:50 2011
@@ -0,0 +1,81 @@
+PREHOOK: query: EXPLAIN
+SELECT NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ),
+ NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ).F2
+ FROM src LIMIT 1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ),
+ NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ).F2
+ FROM src LIMIT 1
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT
(TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION
NAMED_STRUCT (TOK_FUNCTION IF (TOK_FUNCTION ARRAY_CONTAINS (TOK_FUNCTION ARRAY
1 2) 3) "F1" "B1") 1 (TOK_FUNCTION IF (TOK_FUNCTION ARRAY_CONTAINS
(TOK_FUNCTION MAP_KEYS (TOK_FUNCTION MAP "b" "x")) "b") "F2" "B2") 2))
(TOK_SELEXPR (. (TOK_FUNCTION NAMED_STRUCT (TOK_FUNCTION IF (TOK_FUNCTION
ARRAY_CONTAINS (TOK_FUNCTION ARRAY 1 2) 3) "F1" "B1") 1 (TOK_FUNCTION IF
(TOK_FUNCTION ARRAY_CONTAINS (TOK_FUNCTION MAP_KEYS (TOK_FUNCTION MAP "b" "x"))
"b") "F2" "B2") 2) F2))) (TOK_LIMIT 1)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: named_struct(if(array_contains(array(1,2), 3), 'F1',
'B1'),1,if(array_contains(map_keys(map('b':'x')), 'b'), 'F2', 'B2'),2)
+ type: struct<b1:int,f2:int>
+ expr: named_struct(if(array_contains(array(1,2), 3), 'F1',
'B1'),1,if(array_contains(map_keys(map('b':'x')), 'b'), 'F2', 'B2'),2).F2
+ type: int
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 1
+
+
+PREHOOK: query: SELECT NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ),
+ NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ).F2
+ FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output:
file:/var/folders/C4/C40caRNsEM4C4yVangruonVUe7Y/-Tmp-/jonchang/hive_2011-09-27_14-38-28_158_1791133651106020479/-mr-10000
+POSTHOOK: query: SELECT NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ),
+ NAMED_STRUCT(
+ IF(ARRAY_CONTAINS(ARRAY(1, 2), 3), "F1", "B1"), 1,
+ IF(ARRAY_CONTAINS(MAP_KEYS(MAP("b", "x")), "b"), "F2", "B2"), 2
+ ).F2
+ FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output:
file:/var/folders/C4/C40caRNsEM4C4yVangruonVUe7Y/-Tmp-/jonchang/hive_2011-09-27_14-38-28_158_1791133651106020479/-mr-10000
+{"b1":1,"f2":2} 2
Modified: hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/str_to_map.q.out Tue Oct 11
03:36:50 2011
@@ -48,11 +48,11 @@ STAGE PLANS:
PREHOOK: query: select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-41_630_3186988539263498425/-mr-10000
+PREHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-24-51_124_7056362201665426304/-mr-10000
POSTHOOK: query: select str_to_map('a=1,b=2,c=3',',','=')['a'] from src limit 3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-41_630_3186988539263498425/-mr-10000
+POSTHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-24-51_124_7056362201665426304/-mr-10000
1
1
1
@@ -95,14 +95,14 @@ STAGE PLANS:
PREHOOK: query: select str_to_map('a:1,b:2,c:3') from src limit 3
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-45_084_2380690793062196557/-mr-10000
+PREHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-24-56_651_6773854971430318122/-mr-10000
POSTHOOK: query: select str_to_map('a:1,b:2,c:3') from src limit 3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-45_084_2380690793062196557/-mr-10000
-{"b":"2","c":"3","a":"1"}
-{"b":"2","c":"3","a":"1"}
-{"b":"2","c":"3","a":"1"}
+POSTHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-24-56_651_6773854971430318122/-mr-10000
+{"b":"2","a":"1","c":"3"}
+{"b":"2","a":"1","c":"3"}
+{"b":"2","a":"1","c":"3"}
PREHOOK: query: explain select str_to_map('a:1,b:2,c:3',',',':') from src
limit 3
PREHOOK: type: QUERY
POSTHOOK: query: explain select str_to_map('a:1,b:2,c:3',',',':') from src
limit 3
@@ -142,14 +142,14 @@ STAGE PLANS:
PREHOOK: query: select str_to_map('a:1,b:2,c:3',',',':') from src limit 3
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-48_380_2768579016303223096/-mr-10000
+PREHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-25-01_804_3299719471006117768/-mr-10000
POSTHOOK: query: select str_to_map('a:1,b:2,c:3',',',':') from src limit 3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-48_380_2768579016303223096/-mr-10000
-{"b":"2","c":"3","a":"1"}
-{"b":"2","c":"3","a":"1"}
-{"b":"2","c":"3","a":"1"}
+POSTHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-25-01_804_3299719471006117768/-mr-10000
+{"b":"2","a":"1","c":"3"}
+{"b":"2","a":"1","c":"3"}
+{"b":"2","a":"1","c":"3"}
PREHOOK: query: explain select str_to_map(t.ss,',',':')['a']
from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t
limit 3
@@ -205,13 +205,13 @@ from (select transform('a:1,b:2,c:3') us
limit 3
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-51_684_230364883876108478/-mr-10000
+PREHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-25-06_779_3938364123634391474/-mr-10000
POSTHOOK: query: select str_to_map(t.ss,',',':')['a']
from (select transform('a:1,b:2,c:3') using 'cat' as (ss) from src) t
limit 3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-51_684_230364883876108478/-mr-10000
+POSTHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-25-06_779_3938364123634391474/-mr-10000
1
1
1
@@ -229,11 +229,11 @@ POSTHOOK: Output: default@tbl_s2m
PREHOOK: query: select str_to_map(t,'_','=')['333'] from tbl_s2m
PREHOOK: type: QUERY
PREHOOK: Input: default@tbl_s2m
-PREHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-58_864_5131196835428716204/-mr-10000
+PREHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-25-17_345_8328527994867118991/-mr-10000
POSTHOOK: query: select str_to_map(t,'_','=')['333'] from tbl_s2m
POSTHOOK: type: QUERY
POSTHOOK: Input: default@tbl_s2m
-POSTHOOK: Output:
file:/tmp/sdong/hive_2011-02-10_17-23-58_864_5131196835428716204/-mr-10000
+POSTHOOK: Output:
file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-10-05_16-25-17_345_8328527994867118991/-mr-10000
444
444
444
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
Tue Oct 11 03:36:50 2011
@@ -204,6 +204,12 @@ public final class ObjectInspectorFactor
return result;
}
+ public static StandardConstantListObjectInspector
getStandardConstantListObjectInspector(
+ ObjectInspector listElementObjectInspector, List<?> constantValue) {
+ return new StandardConstantListObjectInspector(listElementObjectInspector,
constantValue);
+ }
+
+
static HashMap<List<ObjectInspector>, StandardMapObjectInspector>
cachedStandardMapObjectInspector = new HashMap<List<ObjectInspector>,
StandardMapObjectInspector>();
public static StandardMapObjectInspector getStandardMapObjectInspector(
@@ -222,6 +228,14 @@ public final class ObjectInspectorFactor
return result;
}
+ public static StandardConstantMapObjectInspector
getStandardConstantMapObjectInspector(
+ ObjectInspector mapKeyObjectInspector,
+ ObjectInspector mapValueObjectInspector,
+ Map<?, ?> constantValue) {
+ return new StandardConstantMapObjectInspector(mapKeyObjectInspector,
+ mapValueObjectInspector, constantValue);
+ }
+
static HashMap<List<ObjectInspector>, StandardUnionObjectInspector>
cachedStandardUnionObjectInspector =
new HashMap<List<ObjectInspector>, StandardUnionObjectInspector>();
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
Tue Oct 11 03:36:50 2011
@@ -31,8 +31,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveWritableObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
@@ -74,6 +76,21 @@ public final class ObjectInspectorUtils
}
/**
+ * Ensures that an ObjectInspector is Writable.
+ */
+ public static ObjectInspector getWritableObjectInspector(ObjectInspector oi)
{
+ // All non-primitive OIs are writable so we need only check this case.
+ if (oi.getCategory() == Category.PRIMITIVE) {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
+ if (!(poi instanceof AbstractPrimitiveWritableObjectInspector)) {
+ return PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableObjectInspector(poi.getPrimitiveCategory());
+ }
+ }
+ return oi;
+ }
+
+ /**
* Get the corresponding standard ObjectInspector for an ObjectInspector.
*
* The returned ObjectInspector can be used to inspect the standard object.
@@ -850,6 +867,55 @@ public final class ObjectInspectorUtils
throw new RuntimeException("Unknown category encountered: " + c1);
}
+ public static ConstantObjectInspector
getConstantObjectInspector(ObjectInspector oi, Object value) {
+ ObjectInspector writableOI = getStandardObjectInspector(oi,
ObjectInspectorCopyOption.WRITABLE);
+ Object writableValue =
+ ObjectInspectorConverters.getConverter(oi, writableOI).convert(value);
+ switch (writableOI.getCategory()) {
+ case PRIMITIVE:
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
+ return
PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
+ poi.getPrimitiveCategory(), writableValue);
+ case LIST:
+ ListObjectInspector loi = (ListObjectInspector) oi;
+ return ObjectInspectorFactory.getStandardConstantListObjectInspector(
+ getStandardObjectInspector(
+ loi.getListElementObjectInspector(),
+ ObjectInspectorCopyOption.WRITABLE
+ ),
+ (List<?>)writableValue);
+ case MAP:
+ MapObjectInspector moi = (MapObjectInspector) oi;
+ return ObjectInspectorFactory.getStandardConstantMapObjectInspector(
+ getStandardObjectInspector(
+ moi.getMapKeyObjectInspector(),
+ ObjectInspectorCopyOption.WRITABLE
+ ),
+ getStandardObjectInspector(
+ moi.getMapValueObjectInspector(),
+ ObjectInspectorCopyOption.WRITABLE
+ ),
+ (Map<?, ?>)writableValue);
+ default:
+ throw new IllegalArgumentException(
+ writableOI.getCategory() + " not yet supported for constant OI");
+ }
+ }
+
+ public static boolean supportsConstantObjectInspector(ObjectInspector oi) {
+ switch (oi.getCategory()) {
+ case PRIMITIVE:
+ case LIST:
+ case MAP:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ public static boolean isConstantObjectInspector(ObjectInspector oi) {
+ return (oi instanceof ConstantObjectInspector);
+ }
private ObjectInspectorUtils() {
// prevent instantiation
Added:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantListObjectInspector.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantListObjectInspector.java?rev=1181615&view=auto
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantListObjectInspector.java
(added)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantListObjectInspector.java
Tue Oct 11 03:36:50 2011
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.List;
+
+/**
+ * A StandardListObjectInspector which also implements the
+ * ConstantObjectInspector interface.
+ *
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects,
+ * instead of directly creating an instance of this class.
+ */
+public class StandardConstantListObjectInspector extends
StandardListObjectInspector
+ implements ConstantObjectInspector {
+
+ private List<?> value;
+
+ /**
+ * Call ObjectInspectorFactory.getStandardListObjectInspector instead.
+ */
+ protected StandardConstantListObjectInspector(
+ ObjectInspector listElementObjectInspector, List<?> value) {
+ super(listElementObjectInspector);
+ this.value = value;
+ }
+
+ @Override
+ public List<?> getWritableConstantValue() {
+ return value;
+ }
+}
Added:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantMapObjectInspector.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantMapObjectInspector.java?rev=1181615&view=auto
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantMapObjectInspector.java
(added)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardConstantMapObjectInspector.java
Tue Oct 11 03:36:50 2011
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Map;
+
+/**
+ * A StandardMapObjectInspector which also implements the
+ * ConstantObjectInspector interface.
+ *
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects,
+ * instead of directly creating an instance of this class.
+ */
+public class StandardConstantMapObjectInspector extends
StandardMapObjectInspector
+ implements ConstantObjectInspector {
+
+ private Map<?, ?> value;
+
+ /**
+ * Call ObjectInspectorFactory.getStandardMapObjectInspector instead.
+ */
+ protected StandardConstantMapObjectInspector(ObjectInspector
mapKeyObjectInspector,
+ ObjectInspector mapValueObjectInspector, Map<?, ?> value) {
+ super(mapKeyObjectInspector, mapValueObjectInspector);
+ this.value = value;
+ }
+
+ @Override
+ public Map<?, ?> getWritableConstantValue() {
+ return value;
+ }
+}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java
Tue Oct 11 03:36:50 2011
@@ -26,12 +26,13 @@ import org.apache.hadoop.hive.serde2.obj
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.io.Text;
/**
@@ -187,8 +188,10 @@ public final class PrimitiveObjectInspec
return new WritableConstantDoubleObjectInspector((DoubleWritable)value);
case STRING:
return new WritableConstantStringObjectInspector((Text)value);
+ case TIMESTAMP:
+ return new
WritableConstantTimestampObjectInspector((TimestampWritable)value);
case VOID:
- return new WritableConstantVoidObjectInspector();
+ return new WritableVoidObjectInspector();
default:
throw new RuntimeException("Internal error: Cannot find "
+ "ConstantObjectInspector for " + primitiveCategory);
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantByteObjectInspector.java
Tue Oct 11 03:36:50 2011
@@ -19,7 +19,7 @@ package org.apache.hadoop.hive.serde2.ob
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
-import org.apache.hadoop.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
/**
* A WritableConstantByteObjectInspector is a WritableByteObjectInspector
Added:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java?rev=1181615&view=auto
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java
(added)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantTimestampObjectInspector.java
Tue Oct 11 03:36:50 2011
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector.primitive;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+
+/**
+ * A WritableConstantTimestampObjectInspector is a
WritableTimestampObjectInspector
+ * that implements ConstantObjectInspector.
+ */
+public class WritableConstantTimestampObjectInspector extends
+ WritableTimestampObjectInspector implements
+ ConstantObjectInspector {
+
+ private TimestampWritable value;
+
+ WritableConstantTimestampObjectInspector(TimestampWritable value) {
+ super();
+ this.value = value;
+ }
+
+ @Override
+ public TimestampWritable getWritableConstantValue() {
+ return value;
+ }
+}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java?rev=1181615&r1=1181614&r2=1181615&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java
Tue Oct 11 03:36:50 2011
@@ -17,11 +17,15 @@
*/
package org.apache.hadoop.hive.serde2.objectinspector.primitive;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+
/**
* A WritableVoidObjectInspector inspects a NullWritable Object.
+ * Note that this is also a constant object inspector.
*/
public class WritableVoidObjectInspector extends
- AbstractPrimitiveWritableObjectInspector implements VoidObjectInspector {
+ AbstractPrimitiveWritableObjectInspector implements
+ VoidObjectInspector, ConstantObjectInspector {
WritableVoidObjectInspector() {
super(PrimitiveObjectInspectorUtils.voidTypeEntry);
@@ -33,6 +37,11 @@ public class WritableVoidObjectInspector
}
@Override
+ public Object getWritableConstantValue() {
+ return null;
+ }
+
+ @Override
public Object getPrimitiveJavaObject(Object o) {
throw new RuntimeException("Internal error: cannot create Void object.");
}
