Repository: hive Updated Branches: refs/heads/master 9b2badc3e -> 707bca7ae
HIVE-15351: Disable vectorized VectorUDFAdaptor usage with non-column or constant parameters (Matt McCline, reviewed by Prasanth Jayachandran) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/707bca7a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/707bca7a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/707bca7a Branch: refs/heads/master Commit: 707bca7ae5c3fb6a7a350ca2a89ebbf874f0de52 Parents: 9b2badc Author: Matt McCline <[email protected]> Authored: Mon Dec 5 17:48:54 2016 -0800 Committer: Matt McCline <[email protected]> Committed: Mon Dec 5 17:48:54 2016 -0800 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../ql/exec/vector/VectorizationContext.java | 9 +- .../queries/clientpositive/vector_coalesce_3.q | 19 +++ .../clientpositive/constprog_when_case.q.out | 1 - ql/src/test/results/clientpositive/foldts.q.out | 2 - .../llap/vector_adaptor_usage_mode.q.out | 2 +- .../llap/vector_between_columns.q.out | 2 +- .../clientpositive/llap/vector_coalesce_2.q.out | 2 +- .../clientpositive/llap/vector_coalesce_3.q.out | 127 +++++++++++++++++++ .../llap/vector_when_case_null.q.out | 2 +- .../clientpositive/llap/vectorized_case.q.out | 4 +- .../clientpositive/llap/vectorized_casts.q.out | 2 +- .../clientpositive/spark/vectorized_case.q.out | 2 - .../clientpositive/vector_between_columns.q.out | 1 - .../clientpositive/vector_coalesce_2.q.out | 1 - .../clientpositive/vector_coalesce_3.q.out | 123 ++++++++++++++++++ .../vector_decimal_math_funcs.q.out | 1 - .../clientpositive/vector_when_case_null.q.out | 1 - .../clientpositive/vectorized_case.q.out | 2 - .../clientpositive/vectorized_casts.q.out | 1 - .../clientpositive/vectorized_math_funcs.q.out | 1 - .../vectorized_timestamp_ints_casts.q.out | 2 - 22 files changed, 284 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 5db98f1..aa3d72d 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -287,6 +287,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\ vector_char_simple.q,\ vector_coalesce.q,\ vector_coalesce_2.q,\ + vector_coalesce_3.q,\ vector_complex_all.q,\ vector_complex_join.q,\ vector_count.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index f088941..d9acdf5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -2169,20 +2169,25 @@ public class VectorizationContext { for (int i = 0; i < childExprList.size(); i++) { ExprNodeDesc child = childExprList.get(i); + /* + UNDONE: Until we fix scratch column allocation to not release after each expression, we + UNDONE: cannot have another other than a column or constant in the parameter list. if (child instanceof ExprNodeGenericFuncDesc) { VectorExpression e = getVectorExpression(child, VectorExpressionDescriptor.Mode.PROJECTION); vectorExprs.add(e); variableArgPositions.add(i); exprResultColumnNums.add(e.getOutputColumn()); argDescs[i].setVariable(e.getOutputColumn()); - } else if (child instanceof ExprNodeColumnDesc) { + } else + */ + if (child instanceof ExprNodeColumnDesc) { variableArgPositions.add(i); argDescs[i].setVariable(getInputColumnIndex(((ExprNodeColumnDesc) child).getColumn())); } else if (child instanceof ExprNodeConstantDesc) { // this is a constant (or null) argDescs[i].setConstant((ExprNodeConstantDesc) child); } else { - throw new HiveException("Unable to vectorize custom UDF. Encountered unsupported expr desc : " + throw new HiveException("Unable to use the VectorUDFAdaptor. Encountered unsupported expr desc : " + child); } } http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/queries/clientpositive/vector_coalesce_3.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/vector_coalesce_3.q b/ql/src/test/queries/clientpositive/vector_coalesce_3.q new file mode 100644 index 0000000..e3d9f0a --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_coalesce_3.q @@ -0,0 +1,19 @@ +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; +SET hive.vectorized.execution.enabled=true; +set hive.fetch.task.conversion=none; +SET hive.auto.convert.join=true; + +CREATE TABLE test_1 (member BIGINT, attr BIGINT) STORED AS ORC; + +CREATE TABLE test_2 (member BIGINT) STORED AS ORC; + +INSERT INTO test_1 VALUES (3,1),(2,2); +INSERT INTO test_2 VALUES (1),(2),(3),(4); + +EXPLAIN +SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member; + +SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member; http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/constprog_when_case.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/constprog_when_case.q.out b/ql/src/test/results/clientpositive/constprog_when_case.q.out index 4d2214a..1c8d105 100644 --- a/ql/src/test/results/clientpositive/constprog_when_case.q.out +++ b/ql/src/test/results/clientpositive/constprog_when_case.q.out @@ -56,7 +56,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/foldts.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/foldts.q.out b/ql/src/test/results/clientpositive/foldts.q.out index 4c78495..61714d1 100644 --- a/ql/src/test/results/clientpositive/foldts.q.out +++ b/ql/src/test/results/clientpositive/foldts.q.out @@ -87,7 +87,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator @@ -135,7 +134,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out index 1f23d44..960433d 100644 --- a/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out @@ -928,7 +928,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Execution mode: vectorized, llap + Execution mode: llap LLAP IO: all inputs Reducer 2 Execution mode: vectorized, llap http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out b/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out index 3de8cc4..c62a415 100644 --- a/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_between_columns.q.out @@ -118,7 +118,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized, llap + Execution mode: llap LLAP IO: all inputs Map 2 Map Operator Tree: http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out index b390bfd..5abd3d0 100644 --- a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out @@ -207,7 +207,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 510 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Execution mode: vectorized, llap + Execution mode: llap LLAP IO: all inputs Reducer 2 Execution mode: vectorized, llap http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out new file mode 100644 index 0000000..1a71985 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_coalesce_3.q.out @@ -0,0 +1,127 @@ +PREHOOK: query: CREATE TABLE test_1 (member BIGINT, attr BIGINT) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_1 +POSTHOOK: query: CREATE TABLE test_1 (member BIGINT, attr BIGINT) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_1 +PREHOOK: query: CREATE TABLE test_2 (member BIGINT) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_2 +POSTHOOK: query: CREATE TABLE test_2 (member BIGINT) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_2 +PREHOOK: query: INSERT INTO test_1 VALUES (3,1),(2,2) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@test_1 +POSTHOOK: query: INSERT INTO test_1 VALUES (3,1),(2,2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@test_1 +POSTHOOK: Lineage: test_1.attr EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: test_1.member EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO test_2 VALUES (1),(2),(3),(4) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@test_2 +POSTHOOK: query: INSERT INTO test_2 VALUES (1),(2),(3),(4) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@test_2 +POSTHOOK: Lineage: test_2.member EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: EXPLAIN +SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: m + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: member (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: bigint) + 1 _col0 (type: bigint) + outputColumnNames: _col0, _col2 + input vertices: + 1 Map 2 + Statistics: Num rows: 4 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), CASE WHEN ((COALESCE(_col2,5) > 1)) THEN (_col2) ELSE (null) END (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: all inputs + Map 2 + Map Operator Tree: + TableScan + alias: n + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: member (type: bigint), attr (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Map-reduce partition columns: _col0 (type: bigint) + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: all inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member +PREHOOK: type: QUERY +PREHOOK: Input: default@test_1 +PREHOOK: Input: default@test_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_1 +POSTHOOK: Input: default@test_2 +#### A masked pattern was here #### +1 NULL +2 2 +3 NULL +4 NULL http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out b/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out index 5a3cfe4..d7e4e16 100644 --- a/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out @@ -59,7 +59,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Execution mode: vectorized, llap + Execution mode: llap LLAP IO: all inputs Reducer 2 Execution mode: vectorized, llap http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/llap/vectorized_case.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vectorized_case.q.out b/ql/src/test/results/clientpositive/llap/vectorized_case.q.out index 6e13369..fee510b 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_case.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_case.q.out @@ -62,7 +62,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized, llap + Execution mode: llap LLAP IO: all inputs Stage: Stage-0 @@ -177,7 +177,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized, llap + Execution mode: llap LLAP IO: all inputs Stage: Stage-0 http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out b/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out index a95702d..bcddce5 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_casts.q.out @@ -184,7 +184,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized, llap + Execution mode: llap LLAP IO: all inputs Stage: Stage-0 http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out index c06ea94..9289b1c 100644 --- a/ql/src/test/results/clientpositive/spark/vectorized_case.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorized_case.q.out @@ -62,7 +62,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator @@ -176,7 +175,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/vector_between_columns.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_between_columns.q.out b/ql/src/test/results/clientpositive/vector_between_columns.q.out index 1366e31..a56f2d3 100644 --- a/ql/src/test/results/clientpositive/vector_between_columns.q.out +++ b/ql/src/test/results/clientpositive/vector_between_columns.q.out @@ -131,7 +131,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Local Work: Map Reduce Local Work http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/vector_coalesce_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_coalesce_2.q.out b/ql/src/test/results/clientpositive/vector_coalesce_2.q.out index 56d39d9..1016699 100644 --- a/ql/src/test/results/clientpositive/vector_coalesce_2.q.out +++ b/ql/src/test/results/clientpositive/vector_coalesce_2.q.out @@ -186,7 +186,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 4 Data size: 510 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/vector_coalesce_3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_coalesce_3.q.out b/ql/src/test/results/clientpositive/vector_coalesce_3.q.out new file mode 100644 index 0000000..797da74 --- /dev/null +++ b/ql/src/test/results/clientpositive/vector_coalesce_3.q.out @@ -0,0 +1,123 @@ +PREHOOK: query: CREATE TABLE test_1 (member BIGINT, attr BIGINT) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_1 +POSTHOOK: query: CREATE TABLE test_1 (member BIGINT, attr BIGINT) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_1 +PREHOOK: query: CREATE TABLE test_2 (member BIGINT) STORED AS ORC +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_2 +POSTHOOK: query: CREATE TABLE test_2 (member BIGINT) STORED AS ORC +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_2 +PREHOOK: query: INSERT INTO test_1 VALUES (3,1),(2,2) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@test_1 +POSTHOOK: query: INSERT INTO test_1 VALUES (3,1),(2,2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@test_1 +POSTHOOK: Lineage: test_1.attr EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: test_1.member EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: INSERT INTO test_2 VALUES (1),(2),(3),(4) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@test_2 +POSTHOOK: query: INSERT INTO test_2 VALUES (1),(2),(3),(4) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@test_2 +POSTHOOK: Lineage: test_2.member EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: EXPLAIN +SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_1:n + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_1:n + TableScan + alias: n + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: member (type: bigint), attr (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2 Data size: 32 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + keys: + 0 _col0 (type: bigint) + 1 _col0 (type: bigint) + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: m + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: member (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + keys: + 0 _col0 (type: bigint) + 1 _col0 (type: bigint) + outputColumnNames: _col0, _col2 + Statistics: Num rows: 4 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint), CASE WHEN ((COALESCE(_col2,5) > 1)) THEN (_col2) ELSE (null) END (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 4 Data size: 35 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 35 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member +PREHOOK: type: QUERY +PREHOOK: Input: default@test_1 +PREHOOK: Input: default@test_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT m.member, (CASE WHEN COALESCE(n.attr, 5)>1 THEN n.attr END) AS attr +FROM test_2 m LEFT JOIN test_1 n ON m.member = n.member +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_1 +POSTHOOK: Input: default@test_2 +#### A masked pattern was here #### +1 NULL +2 2 +3 NULL +4 NULL http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/vector_decimal_math_funcs.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_decimal_math_funcs.q.out b/ql/src/test/results/clientpositive/vector_decimal_math_funcs.q.out index bce8f4f..701e0ce 100644 --- a/ql/src/test/results/clientpositive/vector_decimal_math_funcs.q.out +++ b/ql/src/test/results/clientpositive/vector_decimal_math_funcs.q.out @@ -115,7 +115,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/vector_when_case_null.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_when_case_null.q.out b/ql/src/test/results/clientpositive/vector_when_case_null.q.out index 5c981fb..df4bdf3 100644 --- a/ql/src/test/results/clientpositive/vector_when_case_null.q.out +++ b/ql/src/test/results/clientpositive/vector_when_case_null.q.out @@ -53,7 +53,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/vectorized_case.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vectorized_case.q.out b/ql/src/test/results/clientpositive/vectorized_case.q.out index 08c1412..b0166f2 100644 --- a/ql/src/test/results/clientpositive/vectorized_case.q.out +++ b/ql/src/test/results/clientpositive/vectorized_case.q.out @@ -59,7 +59,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator @@ -170,7 +169,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/vectorized_casts.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vectorized_casts.q.out b/ql/src/test/results/clientpositive/vectorized_casts.q.out index 8520125..e39bff4 100644 --- a/ql/src/test/results/clientpositive/vectorized_casts.q.out +++ b/ql/src/test/results/clientpositive/vectorized_casts.q.out @@ -181,7 +181,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out b/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out index 7b7dedc..c967dde 100644 --- a/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out +++ b/ql/src/test/results/clientpositive/vectorized_math_funcs.q.out @@ -131,7 +131,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator http://git-wip-us.apache.org/repos/asf/hive/blob/707bca7a/ql/src/test/results/clientpositive/vectorized_timestamp_ints_casts.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vectorized_timestamp_ints_casts.q.out b/ql/src/test/results/clientpositive/vectorized_timestamp_ints_casts.q.out index 31afb2c..c7f2a74 100644 --- a/ql/src/test/results/clientpositive/vectorized_timestamp_ints_casts.q.out +++ b/ql/src/test/results/clientpositive/vectorized_timestamp_ints_casts.q.out @@ -59,7 +59,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator @@ -192,7 +191,6 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Execution mode: vectorized Stage: Stage-0 Fetch Operator
