HIVE-12206: ClassNotFound Exception during query compilation with Tez and Union query and GenericUDFs (Jason Dere, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fe6ebf77 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fe6ebf77 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fe6ebf77 Branch: refs/heads/master-fixed Commit: fe6ebf77a5494b737479af3f1159e46ed6aa9d24 Parents: 7073ce3 Author: Jason Dere <[email protected]> Authored: Tue Nov 3 15:38:31 2015 -0800 Committer: Jason Dere <[email protected]> Committed: Tue Nov 3 15:38:31 2015 -0800 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../hive/udf/example/GenericUDFExampleAdd.java | 48 ++++++++++++++++++++ .../apache/hadoop/hive/ql/exec/Utilities.java | 2 + .../queries/clientpositive/tez_union_with_udf.q | 13 ++++++ .../clientpositive/tez/tez_union_with_udf.q.out | 36 +++++++++++++++ 5 files changed, 100 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 7416d32..2d1d274 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -424,6 +424,7 @@ minillap.query.files=bucket_map_join_tez1.q,\ tez_union_view.q,\ tez_union_decimal.q,\ tez_union_group_by.q,\ + tez_union_with_udf.q,\ tez_smb_main.q,\ tez_smb_1.q,\ vectorized_dynamic_partition_pruning.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/itests/test-serde/src/main/java/org/apache/hadoop/hive/udf/example/GenericUDFExampleAdd.java ---------------------------------------------------------------------- diff --git a/itests/test-serde/src/main/java/org/apache/hadoop/hive/udf/example/GenericUDFExampleAdd.java b/itests/test-serde/src/main/java/org/apache/hadoop/hive/udf/example/GenericUDFExampleAdd.java new file mode 100644 index 0000000..85906c2 --- /dev/null +++ b/itests/test-serde/src/main/java/org/apache/hadoop/hive/udf/example/GenericUDFExampleAdd.java @@ -0,0 +1,48 @@ +package org.apache.hadoop.hive.udf.example; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +public class GenericUDFExampleAdd extends GenericUDF { + + Converter converter0; + Converter converter1; + DoubleWritable result = new DoubleWritable(); + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) + throws UDFArgumentException { + ObjectInspector doubleOI = PrimitiveObjectInspectorFactory + .getPrimitiveWritableObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.DOUBLE); + converter0 = (Converter) ObjectInspectorConverters.getConverter(arguments[0], doubleOI); + converter1 = (Converter) ObjectInspectorConverters.getConverter(arguments[1], doubleOI); + return doubleOI; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + DoubleWritable dw0 = (DoubleWritable) converter0.convert(arguments[0].get()); + DoubleWritable dw1 = (DoubleWritable) converter1.convert(arguments[0].get()); + if (dw0 == null || dw1 == null) { + return null; + } + result.set(dw0.get() + dw1.get()); + return result; + } + + @Override + public String getDisplayString(String[] children) { + return "GenericUDFExampleAdd"; + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 665b3f7..02adf0c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -1057,6 +1057,7 @@ public final class Utilities { */ private static void serializeObjectByKryo(Kryo kryo, Object plan, OutputStream out) { Output output = new Output(out); + kryo.setClassLoader(getSessionSpecifiedClassLoader()); kryo.writeObject(output, plan); output.close(); } @@ -1080,6 +1081,7 @@ public final class Utilities { private static <T> T deserializeObjectByKryo(Kryo kryo, InputStream in, Class<T> clazz ) { Input inp = new Input(in); + kryo.setClassLoader(getSessionSpecifiedClassLoader()); T t = kryo.readObject(inp,clazz); inp.close(); return t; http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/ql/src/test/queries/clientpositive/tez_union_with_udf.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/tez_union_with_udf.q b/ql/src/test/queries/clientpositive/tez_union_with_udf.q new file mode 100644 index 0000000..6826530 --- /dev/null +++ b/ql/src/test/queries/clientpositive/tez_union_with_udf.q @@ -0,0 +1,13 @@ +select * from (select key + key from src limit 1) a +union all +select * from (select key + key from src limit 1) b; + + +add jar ${system:maven.local.repository}/org/apache/hive/hive-it-test-serde/${system:hive.version}/hive-it-test-serde-${system:hive.version}.jar; + +create temporary function example_add as 'org.apache.hadoop.hive.udf.example.GenericUDFExampleAdd'; + +-- Now try the query with the UDF +select example_add(key, key)from (select key from src limit 1) a +union all +select example_add(key, key)from (select key from src limit 1) b; http://git-wip-us.apache.org/repos/asf/hive/blob/fe6ebf77/ql/src/test/results/clientpositive/tez/tez_union_with_udf.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/tez_union_with_udf.q.out b/ql/src/test/results/clientpositive/tez/tez_union_with_udf.q.out new file mode 100644 index 0000000..923e098 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/tez_union_with_udf.q.out @@ -0,0 +1,36 @@ +PREHOOK: query: select * from (select key + key from src limit 1) a +union all +select * from (select key + key from src limit 1) b +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select * from (select key + key from src limit 1) a +union all +select * from (select key + key from src limit 1) b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +476.0 +476.0 +PREHOOK: query: create temporary function example_add as 'org.apache.hadoop.hive.udf.example.GenericUDFExampleAdd' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: example_add +POSTHOOK: query: create temporary function example_add as 'org.apache.hadoop.hive.udf.example.GenericUDFExampleAdd' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: example_add +PREHOOK: query: -- Now try the query with the UDF +select example_add(key, key)from (select key from src limit 1) a +union all +select example_add(key, key)from (select key from src limit 1) b +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- Now try the query with the UDF +select example_add(key, key)from (select key from src limit 1) a +union all +select example_add(key, key)from (select key from src limit 1) b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +476.0 +476.0
