HIVE-12857 : LLAP: modify the decider to allow using LLAP with whitelisted UDFs (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e44198f5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e44198f5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e44198f5 Branch: refs/heads/llap Commit: e44198f5e6b2b5a83c34d70bd12bb4e772f737dd Parents: be00006 Author: Sergey Shelukhin <[email protected]> Authored: Thu Feb 25 19:41:56 2016 -0800 Committer: Sergey Shelukhin <[email protected]> Committed: Thu Feb 25 19:41:56 2016 -0800 ---------------------------------------------------------------------- itests/qtest/pom.xml | 4 +- .../test/resources/testconfiguration.properties | 6 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 15 + .../hadoop/hive/ql/exec/FunctionTask.java | 10 +- .../apache/hadoop/hive/ql/exec/Registry.java | 44 +++ .../hive/ql/optimizer/physical/LlapDecider.java | 50 +-- ql/src/test/queries/clientpositive/llap_udf.q | 47 +++ .../results/clientpositive/llap/llap_udf.q.out | 307 +++++++++++++++++++ 8 files changed, 458 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/itests/qtest/pom.xml ---------------------------------------------------------------------- diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index 292e562..cb68e39 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -431,7 +431,7 @@ templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm" queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/" queryFile="${qfile}" - excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files},${spark.only.query.files},${disabled.query.files}" + excludeQueryFile="${minillap.query.files},${minimr.query.files},${minitez.query.files},${encrypted.query.files},${spark.only.query.files},${disabled.query.files}" queryFileRegex="${qfile_regex}" clusterMode="${clustermode}" runDisabled="${run_disabled}" @@ -535,7 +535,7 @@ templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm" queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/" queryFile="${qfile}" - includeQueryFile="${minillap.query.files}" + includeQueryFile="${minillap.query.files},${minillap.shared.query.files}" queryFileRegex="${qfile_regex}" clusterMode="llap" runDisabled="${run_disabled}" http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 58d0a45..68dbd0c 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -445,7 +445,7 @@ minitez.query.files=bucket_map_join_tez1.q,\ -minillap.query.files=bucket_map_join_tez1.q,\ +minillap.shared.query.files=bucket_map_join_tez1.q,\ bucket_map_join_tez2.q,\ constprog_dpp.q,\ cte_1.q,\ @@ -496,6 +496,10 @@ minillap.query.files=bucket_map_join_tez1.q,\ tez_join.q,\ tez_union_multiinsert.q + +minillap.query.files=llap_udf.q + + encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_insert_partition_static.q,\ encryption_insert_partition_dynamic.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index e0e030f..d1e1441 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -1480,6 +1480,21 @@ public final class FunctionRegistry { return system.registerPermanentFunction(functionName, className, registerToSession, resources); } + public static boolean isPermanentFunction(ExprNodeGenericFuncDesc fnExpr) { + GenericUDF udf = fnExpr.getGenericUDF(); + if (udf == null) return false; + + Class<?> clazz = udf.getClass(); + if (udf instanceof GenericUDFBridge) { + clazz = ((GenericUDFBridge)udf).getUdfClass(); + } + + if (clazz != null) { + return system.isPermanentFunc(clazz); + } + return false; + } + public static void unregisterPermanentFunction(String functionName) throws HiveException { system.unregisterFunction(functionName); unregisterTemporaryUDF(functionName); http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java index 77f11b9..1b971fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java @@ -136,8 +136,16 @@ public class FunctionTask extends Task<FunctionWork> { // For permanent functions, check for any resources from local filesystem. checkLocalFunctionResources(db, createFunctionDesc.getResources()); - FunctionInfo registered = FunctionRegistry.registerPermanentFunction( + FunctionInfo registered = null; + try { + registered = FunctionRegistry.registerPermanentFunction( registeredName, className, true, toFunctionResource(resources)); + } catch (RuntimeException ex) { + Throwable t = ex; + while (t.getCause() != null) { + t = t.getCause(); + } + } if (registered == null) { console.printError("Failed to register " + registeredName + " using class " + createFunctionDesc.getClassName()); http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java index 16c8c8f..d5f4a37 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java @@ -55,6 +55,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -71,6 +72,11 @@ public class Registry { */ private final Map<String, FunctionInfo> mFunctions = new LinkedHashMap<String, FunctionInfo>(); private final Set<Class<?>> builtIns = Collections.synchronizedSet(new HashSet<Class<?>>()); + /** + * Persistent map contains refcounts that are only modified in synchronized methods for now, + * so there's no separate effort to make refcount operations thread-safe. + */ + private final Map<Class<?>, Integer> persistent = new ConcurrentHashMap<>(); private final Set<ClassLoader> mSessionUDFLoaders = new LinkedHashSet<ClassLoader>(); private final boolean isNative; @@ -285,6 +291,10 @@ public class Registry { return udfClass != null && builtIns.contains(udfClass); } + public boolean isPermanentFunc(Class<?> udfClass) { + return udfClass != null && persistent.containsKey(udfClass); + } + public synchronized Set<String> getCurrentFunctionNames() { return getFunctionNames((Pattern)null); } @@ -415,9 +425,27 @@ public class Registry { mFunctions.put(functionName, function); if (function.isBuiltIn()) { builtIns.add(function.getFunctionClass()); + } else if (function.isPersistent()) { + Class<?> functionClass = getPermanentUdfClass(function); + Integer refCount = persistent.get(functionClass); + persistent.put(functionClass, Integer.valueOf(refCount == null ? 1 : refCount + 1)); } } + private Class<?> getPermanentUdfClass(FunctionInfo function) { + Class<?> functionClass = function.getFunctionClass(); + if (functionClass == null) { + // Expected for permanent UDFs at this point. + ClassLoader loader = Utilities.getSessionSpecifiedClassLoader(); + try { + functionClass = Class.forName(function.getClassName(), true, loader); + } catch (ClassNotFoundException ex) { + throw new RuntimeException(ex); + } + } + return functionClass; + } + public synchronized void unregisterFunction(String functionName) throws HiveException { functionName = functionName.toLowerCase(); FunctionInfo fi = mFunctions.get(functionName); @@ -427,6 +455,21 @@ public class Registry { } mFunctions.remove(functionName); fi.discarded(); + if (fi.isPersistent()) { + removePersistentFunction(fi); + } + } + } + + /** Should only be called from synchronized methods. */ + private void removePersistentFunction(FunctionInfo fi) { + Class<?> functionClass = getPermanentUdfClass(fi); + Integer refCount = persistent.get(functionClass); + assert refCount != null; + if (refCount == 1) { + persistent.remove(functionClass); + } else { + persistent.put(functionClass, Integer.valueOf(refCount - 1)); } } @@ -517,6 +560,7 @@ public class Registry { } mFunctions.clear(); builtIns.clear(); + persistent.clear(); } public synchronized void closeCUDFLoaders() { http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index f2998af..80dbd31 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -101,15 +101,18 @@ public class LlapDecider implements PhysicalPlanResolver { class LlapDecisionDispatcher implements Dispatcher { private final HiveConf conf; private final boolean doSkipUdfCheck; + private final boolean arePermanentFnsAllowed; public LlapDecisionDispatcher(PhysicalContext pctx) { conf = pctx.getConf(); doSkipUdfCheck = HiveConf.getBoolVar(conf, ConfVars.LLAP_SKIP_COMPILE_UDF_CHECK); + arePermanentFnsAllowed = HiveConf.getBoolVar(conf, ConfVars.LLAP_DAEMON_ALLOW_PERMANENT_FNS); } @Override public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs) throws SemanticException { + @SuppressWarnings("unchecked") Task<? extends Serializable> currTask = (Task<? extends Serializable>) nd; if (currTask instanceof TezTask) { TezWork work = ((TezTask) currTask).getWork(); @@ -237,10 +240,20 @@ public class LlapDecider implements PhysicalPlanResolver { exprs.addAll(cur.getChildren()); } - if (!doSkipUdfCheck && cur instanceof ExprNodeGenericFuncDesc - && !FunctionRegistry.isBuiltInFuncExpr((ExprNodeGenericFuncDesc)cur)) { - LOG.info("Not a built-in function: " + cur.getExprString()); - return false; + if (!doSkipUdfCheck && cur instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc)cur; + boolean isBuiltIn = FunctionRegistry.isBuiltInFuncExpr(funcDesc); + if (!isBuiltIn) { + if (!arePermanentFnsAllowed) { + LOG.info("Not a built-in function: " + cur.getExprString() + + " (permanent functions are disabled)"); + return false; + } + if (!FunctionRegistry.isPermanentFunction(funcDesc)) { + LOG.info("Not a built-in or permanent function: " + cur.getExprString()); + return false; + } + } } } return true; @@ -261,24 +274,22 @@ public class LlapDecider implements PhysicalPlanResolver { } private boolean checkExpressions(Collection<ExprNodeDesc> exprs) { - boolean result = true; - for (ExprNodeDesc expr: exprs) { - result = result && checkExpression(expr); + for (ExprNodeDesc expr : exprs) { + if (!checkExpression(expr)) return false; } - return result; + return true; } private boolean checkAggregators(Collection<AggregationDesc> aggs) { - boolean result = true; try { - for (AggregationDesc agg: aggs) { - result = result && checkAggregator(agg); - } + for (AggregationDesc agg: aggs) { + if (!checkAggregator(agg)) return false; + } } catch (SemanticException e) { - LOG.warn("Exception testing aggregators.",e); - result = false; + LOG.warn("Exception testing aggregators.",e); + return false; } - return result; + return true; } private Map<Rule, NodeProcessor> getRules() { @@ -291,8 +302,7 @@ public class LlapDecider implements PhysicalPlanResolver { return new Boolean(false); } }); - opRules.put(new RuleRegExp("No user code in fil", - FilterOperator.getOperatorName() + "%"), + opRules.put(new RuleRegExp("No user code in fil", FilterOperator.getOperatorName() + "%"), new NodeProcessor() { @Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, @@ -301,8 +311,7 @@ public class LlapDecider implements PhysicalPlanResolver { return new Boolean(checkExpression(expr)); } }); - opRules.put(new RuleRegExp("No user code in gby", - GroupByOperator.getOperatorName() + "%"), + opRules.put(new RuleRegExp("No user code in gby", GroupByOperator.getOperatorName() + "%"), new NodeProcessor() { @Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, @@ -311,8 +320,7 @@ public class LlapDecider implements PhysicalPlanResolver { return new Boolean(checkAggregators(aggs)); } }); - opRules.put(new RuleRegExp("No user code in select", - SelectOperator.getOperatorName() + "%"), + opRules.put(new RuleRegExp("No user code in select", SelectOperator.getOperatorName() + "%"), new NodeProcessor() { @Override public Object process(Node n, Stack<Node> s, NodeProcessorCtx c, http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/test/queries/clientpositive/llap_udf.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/llap_udf.q b/ql/src/test/queries/clientpositive/llap_udf.q new file mode 100644 index 0000000..c964f2b --- /dev/null +++ b/ql/src/test/queries/clientpositive/llap_udf.q @@ -0,0 +1,47 @@ +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; +set hive.execution.mode=llap; +set hive.llap.execution.mode=all; +set hive.fetch.task.conversion=none; +set hive.llap.daemon.allow.permanent.fns=true; + +drop table if exists src_orc; +create table src_orc stored as orc as select * from src; + +-- Not using GenericUDFTestGetJavaBoolean; that is already registered when tests begin + +CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'; + +EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc; + +CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'; +CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'; +CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'; + +EXPLAIN +SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc; + +-- Verification is based on classes, so 0 would work based on 4. +EXPLAIN +SELECT test_udf0(cast(key as string)) from src_orc; + +DROP FUNCTION test_udf2; + +-- ...verify that 3 still works +EXPLAIN +SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc; + +DROP FUNCTION test_udf4; + +-- ...now 0 should stop working +EXPLAIN +SELECT test_udf0(cast(key as string)) from src_orc; + +set hive.llap.daemon.allow.permanent.fns=false; + +EXPLAIN +SELECT test_udf3(cast(key as string)) from src_orc; + + +drop table if exists src_orc; +set hive.execution.mode=container; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/test/results/clientpositive/llap/llap_udf.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/llap_udf.q.out b/ql/src/test/results/clientpositive/llap/llap_udf.q.out new file mode 100644 index 0000000..29ed978 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/llap_udf.q.out @@ -0,0 +1,307 @@ +PREHOOK: query: drop table if exists src_orc +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists src_orc +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_orc stored as orc as select * from src +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src_orc +POSTHOOK: query: create table src_orc stored as orc as select * from src +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_orc +PREHOOK: query: -- Not using GenericUDFTestGetJavaBoolean; that is already registered when tests begin + +CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: test_udf0 +POSTHOOK: query: -- Not using GenericUDFTestGetJavaBoolean; that is already registered when tests begin + +CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: test_udf0 +PREHOOK: query: EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_orc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: evaluate_npe(key) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: database:default +PREHOOK: Output: default.test_udf2 +POSTHOOK: query: CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: database:default +POSTHOOK: Output: default.test_udf2 +PREHOOK: query: CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: database:default +PREHOOK: Output: default.test_udf3 +POSTHOOK: query: CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: database:default +POSTHOOK: Output: default.test_udf3 +PREHOOK: query: CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: database:default +PREHOOK: Output: default.test_udf4 +POSTHOOK: query: CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: database:default +POSTHOOK: Output: default.test_udf4 +PREHOOK: query: EXPLAIN +SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_orc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: GenericUDFTestGetJavaString(key) (type: string), GenericUDFTestGetJavaString(key) (type: string), evaluate_npe(key) (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- Verification is based on classes, so 0 would work based on 4. +EXPLAIN +SELECT test_udf0(cast(key as string)) from src_orc +PREHOOK: type: QUERY +POSTHOOK: query: -- Verification is based on classes, so 0 would work based on 4. +EXPLAIN +SELECT test_udf0(cast(key as string)) from src_orc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_orc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: evaluate_npe(key) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: DROP FUNCTION test_udf2 +PREHOOK: type: DROPFUNCTION +PREHOOK: Output: database:default +PREHOOK: Output: default.test_udf2 +POSTHOOK: query: DROP FUNCTION test_udf2 +POSTHOOK: type: DROPFUNCTION +POSTHOOK: Output: database:default +POSTHOOK: Output: default.test_udf2 +PREHOOK: query: -- ...verify that 3 still works +EXPLAIN +SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc +PREHOOK: type: QUERY +POSTHOOK: query: -- ...verify that 3 still works +EXPLAIN +SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_orc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: GenericUDFTestGetJavaString(key) (type: string), evaluate_npe(key) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: DROP FUNCTION test_udf4 +PREHOOK: type: DROPFUNCTION +PREHOOK: Output: database:default +PREHOOK: Output: default.test_udf4 +POSTHOOK: query: DROP FUNCTION test_udf4 +POSTHOOK: type: DROPFUNCTION +POSTHOOK: Output: database:default +POSTHOOK: Output: default.test_udf4 +PREHOOK: query: -- ...now 0 should stop working +EXPLAIN +SELECT test_udf0(cast(key as string)) from src_orc +PREHOOK: type: QUERY +POSTHOOK: query: -- ...now 0 should stop working +EXPLAIN +SELECT test_udf0(cast(key as string)) from src_orc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_orc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: evaluate_npe(key) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT test_udf3(cast(key as string)) from src_orc +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT test_udf3(cast(key as string)) from src_orc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src_orc + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: GenericUDFTestGetJavaString(key) (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table if exists src_orc +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@src_orc +PREHOOK: Output: default@src_orc +POSTHOOK: query: drop table if exists src_orc +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@src_orc +POSTHOOK: Output: default@src_orc
