Repository: hive Updated Branches: refs/heads/branch-2.0 77c384da4 -> 09e4eb1dd
HIVE-12645: ConstantPropagateProcCtx.resolve() should verify internal names in addition to alias to match 2 columns from different row schemas (Hari Subramaniyan, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/09e4eb1d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/09e4eb1d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/09e4eb1d Branch: refs/heads/branch-2.0 Commit: 09e4eb1dd995a54d7e02515b3d6086edc64e1ba6 Parents: 77c384d Author: Hari Subramaniyan <harisan...@apache.org> Authored: Mon Jan 11 23:09:52 2016 -0800 Committer: Hari Subramaniyan <harisan...@apache.org> Committed: Mon Jan 11 23:09:52 2016 -0800 ---------------------------------------------------------------------- .../ql/optimizer/ConstantPropagateProcCtx.java | 23 ++++++++-- .../correlation/ReduceSinkDeDuplication.java | 13 ++++++ .../results/clientpositive/bucket_groupby.q.out | 12 ++--- .../clientpositive/dynamic_rdd_cache.q.out | 8 ++-- .../dynpart_sort_optimization2.q.out | 12 ++--- .../results/clientpositive/groupby_ppd.q.out | 10 ++-- .../clientpositive/groupby_sort_1_23.q.out | 6 +-- .../clientpositive/groupby_sort_skew_1_23.q.out | 10 ++-- .../list_bucket_query_oneskew_2.q.out | 6 +-- .../clientpositive/partition_multilevels.q.out | 12 ++--- .../results/clientpositive/quotedid_basic.q.out | 18 ++++---- .../clientpositive/quotedid_partition.q.out | 6 +-- .../clientpositive/reducesink_dedup.q.out | 48 ++++++++++---------- .../spark/dynamic_rdd_cache.q.out | 8 ++-- .../spark/groupby_sort_1_23.q.out | 6 +-- .../spark/groupby_sort_skew_1_23.q.out | 10 ++-- .../tez/dynpart_sort_optimization2.q.out | 12 ++--- .../tez/vector_null_projection.q.out | 26 ++++++----- .../clientpositive/vector_null_projection.q.out | 26 ++++++----- 19 files changed, 155 insertions(+), 117 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java index 0a61f12..1814550 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java @@ -172,12 +172,29 @@ public class ConstantPropagateProcCtx implements NodeProcessorCtx { Map<ColumnInfo, ExprNodeDesc> c = opToConstantExprs.get(parent); for (Entry<ColumnInfo, ExprNodeDesc> e : c.entrySet()) { ColumnInfo ci = e.getKey(); - ColumnInfo rci = null; ExprNodeDesc constant = e.getValue(); - rci = resolve(ci, rs, parent.getSchema()); + boolean resolved = false; + ColumnInfo rci = resolve(ci, rs, parent.getSchema()); + if (rci != null) { constants.put(rci, constant); - } else { + resolved = true; + } + if (!resolved && + op.getColumnExprMap() != null && op.getColumnExprMap().entrySet() != null) { + for (Entry<String, ExprNodeDesc> entry : op.getColumnExprMap().entrySet()) { + if (entry.getValue().isSame(constant)) { + ColumnInfo rsColumnInfo = rs.getColumnInfo(entry.getKey()); + if (rsColumnInfo == null) { + continue; + } + constants.put(rsColumnInfo, constant); + resolved = true; + } + } + } + + if (!resolved) { LOG.debug("Can't resolve " + ci.getTabAlias() + "." + ci.getAlias() + "(" + ci.getInternalName() + ") from rs:" + rs); } http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java index 1746dd3..638b91e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.optimizer.Transform; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.PlanUtils; @@ -388,6 +389,18 @@ public class ReduceSinkDeDuplication extends Transform { */ private Integer checkExprs(List<ExprNodeDesc> ckeys, List<ExprNodeDesc> pkeys, ReduceSinkOperator cRS, ReduceSinkOperator pRS) throws SemanticException { + // If ckeys or pkeys have constant node expressions avoid the merge. + for (ExprNodeDesc ck : ckeys) { + if (ck instanceof ExprNodeConstantDesc) { + return null; + } + } + for (ExprNodeDesc pk : pkeys) { + if (pk instanceof ExprNodeConstantDesc) { + return null; + } + } + Integer moveKeyColTo = 0; if (ckeys == null || ckeys.isEmpty()) { if (pkeys != null && !pkeys.isEmpty()) { http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/bucket_groupby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/bucket_groupby.q.out b/ql/src/test/results/clientpositive/bucket_groupby.q.out index 0e30801..676326a 100644 --- a/ql/src/test/results/clientpositive/bucket_groupby.q.out +++ b/ql/src/test/results/clientpositive/bucket_groupby.q.out @@ -398,16 +398,16 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) + key expressions: _col0 (type: string), 3 (type: int) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Map-reduce partition columns: _col0 (type: string), 3 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE TopN Hash Memory Usage: 0.1 value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int) + keys: KEY._col0 (type: string), 3 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -935,15 +935,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int) + key expressions: _col0 (type: string), 3 (type: int) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int) + Map-reduce partition columns: _col0 (type: string), 3 (type: int) Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int) + keys: KEY._col0 (type: string), 3 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out b/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out index 3153c7e..c886481 100644 --- a/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out +++ b/ql/src/test/results/clientpositive/dynamic_rdd_cache.q.out @@ -1067,15 +1067,15 @@ STAGE PLANS: Map Operator Tree: TableScan Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) sort order: ++++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int) + Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,input:int>) Reduce Operator Tree: Group By Operator - aggregations: stddev_samp(VALUE._col0), avg(VALUE._col1) - keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: int) + aggregations: avg(VALUE._col0), stddev_samp(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: int), 3 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out index 24ac550..9409412 100644 --- a/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out @@ -1547,15 +1547,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) + key expressions: 'day' (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map-reduce partition columns: 'day' (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: 'day' (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -1663,15 +1663,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) + key expressions: 'day' (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string) + Map-reduce partition columns: 'day' (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: 'day' (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/groupby_ppd.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/groupby_ppd.q.out b/ql/src/test/results/clientpositive/groupby_ppd.q.out index 6164a26..edc8aa3 100644 --- a/ql/src/test/results/clientpositive/groupby_ppd.q.out +++ b/ql/src/test/results/clientpositive/groupby_ppd.q.out @@ -42,9 +42,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) + key expressions: 1 (type: int), _col1 (type: int) sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Map-reduce partition columns: 1 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE TableScan alias: c @@ -68,13 +68,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int) + key expressions: 1 (type: int), _col1 (type: int) sort order: ++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int) + Map-reduce partition columns: 1 (type: int), _col1 (type: int) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: int), KEY._col1 (type: int) + keys: 1 (type: int), KEY._col1 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out index 7333677..7e264b8 100644 --- a/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out +++ b/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out @@ -1873,9 +1873,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), 1 (type: int), _col2 (type: string) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col3 (type: bigint) @@ -1937,7 +1937,7 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out index e19d1de..2694494 100644 --- a/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out +++ b/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out @@ -1939,7 +1939,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: rand() (type: double) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE @@ -2003,7 +2003,7 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) mode: partials outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE @@ -2031,9 +2031,9 @@ STAGE PLANS: TableScan GatherStats: false Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), 1 (type: int), _col2 (type: string) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col3 (type: bigint) @@ -2067,7 +2067,7 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) mode: final outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out index be77ba8..ab0dd15 100644 --- a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out +++ b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out @@ -700,9 +700,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int) + key expressions: 484 (type: int) sort order: + - Map-reduce partition columns: _col0 (type: int) + Map-reduce partition columns: 484 (type: int) Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col1 (type: bigint) @@ -762,7 +762,7 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: int) + keys: 484 (type: int) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/partition_multilevels.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/partition_multilevels.q.out b/ql/src/test/results/clientpositive/partition_multilevels.q.out index c1c8778..01e675d 100644 --- a/ql/src/test/results/clientpositive/partition_multilevels.q.out +++ b/ql/src/test/results/clientpositive/partition_multilevels.q.out @@ -1000,15 +1000,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + key expressions: '2222' (type: string), _col1 (type: string), _col2 (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Map-reduce partition columns: '2222' (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) + keys: '2222' (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 54 Data size: 573 Basic stats: COMPLETE Column stats: NONE @@ -1588,15 +1588,15 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + key expressions: '2222' (type: string), _col1 (type: string), _col2 (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Map-reduce partition columns: '2222' (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 108 Data size: 1146 Basic stats: COMPLETE Column stats: NONE value expressions: _col3 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) + keys: '2222' (type: string), KEY._col1 (type: string), KEY._col2 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 54 Data size: 573 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/quotedid_basic.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/quotedid_basic.q.out b/ql/src/test/results/clientpositive/quotedid_basic.q.out index 43b63f2..a406148 100644 --- a/ql/src/test/results/clientpositive/quotedid_basic.q.out +++ b/ql/src/test/results/clientpositive/quotedid_basic.q.out @@ -110,13 +110,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), '1' (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string), '1' (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -165,13 +165,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), '1' (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string), '1' (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -273,13 +273,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + key expressions: _col0 (type: string), _col1 (type: string), '1' (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), _col1 (type: string), '1' (type: string) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) + keys: KEY._col0 (type: string), KEY._col1 (type: string), '1' (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/quotedid_partition.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/quotedid_partition.q.out b/ql/src/test/results/clientpositive/quotedid_partition.q.out index d34a005..e01a427 100644 --- a/ql/src/test/results/clientpositive/quotedid_partition.q.out +++ b/ql/src/test/results/clientpositive/quotedid_partition.q.out @@ -55,13 +55,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) + key expressions: '10' (type: string), _col1 (type: string), 'a' (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) + Map-reduce partition columns: '10' (type: string), _col1 (type: string), 'a' (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) + keys: '10' (type: string), KEY._col1 (type: string), 'a' (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/reducesink_dedup.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/reducesink_dedup.q.out b/ql/src/test/results/clientpositive/reducesink_dedup.q.out index 77bffff..b89df52 100644 --- a/ql/src/test/results/clientpositive/reducesink_dedup.q.out +++ b/ql/src/test/results/clientpositive/reducesink_dedup.q.out @@ -10,29 +10,29 @@ distribute by 1 sort by 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -almond azure blanched chiffon midnight -almond aquamarine dodger light gainsboro -almond antique sky peru orange -almond antique medium spring khaki -almond antique blue firebrick mint -almond azure aquamarine papaya violet -almond aquamarine yellow dodger mint -almond aquamarine floral ivory bisque -almond antique violet mint lemon -almond antique gainsboro frosted violet -almond antique olive coral navajo -almond antique misty red olive -almond antique metallic orange dim -almond antique forest lavender goldenrod -almond antique chartreuse khaki white -almond aquamarine sandy cyan gainsboro -almond aquamarine rose maroon antique -almond aquamarine midnight light salmon -almond antique violet turquoise frosted -almond antique violet chocolate turquoise -almond aquamarine pink moccasin thistle -almond aquamarine burnished black steel -almond antique salmon chartreuse burlywood -almond antique chartreuse lavender yellow almond antique burnished rose metallic almond antique burnished rose metallic +almond antique chartreuse lavender yellow +almond antique salmon chartreuse burlywood +almond aquamarine burnished black steel +almond aquamarine pink moccasin thistle +almond antique violet chocolate turquoise +almond antique violet turquoise frosted +almond aquamarine midnight light salmon +almond aquamarine rose maroon antique +almond aquamarine sandy cyan gainsboro +almond antique chartreuse khaki white +almond antique forest lavender goldenrod +almond antique metallic orange dim +almond antique misty red olive +almond antique olive coral navajo +almond antique gainsboro frosted violet +almond antique violet mint lemon +almond aquamarine floral ivory bisque +almond aquamarine yellow dodger mint +almond azure aquamarine papaya violet +almond antique blue firebrick mint +almond antique medium spring khaki +almond antique sky peru orange +almond aquamarine dodger light gainsboro +almond azure blanched chiffon midnight http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out b/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out index 9043fb1..3602e26 100644 --- a/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out +++ b/ql/src/test/results/clientpositive/spark/dynamic_rdd_cache.q.out @@ -1001,16 +1001,16 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int) + key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) sort order: ++++ - Map-reduce partition columns: _col0 (type: int), _col1 (type: int), _col2 (type: string), _col3 (type: int) + Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: int), 3 (type: int) Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE value expressions: _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,input:int>) Reducer 5 Reduce Operator Tree: Group By Operator - aggregations: stddev_samp(VALUE._col0), avg(VALUE._col1) - keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: string), KEY._col3 (type: int) + aggregations: avg(VALUE._col0), stddev_samp(VALUE._col1) + keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: int), 3 (type: int) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 239e803..587a7e0 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -1204,9 +1204,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), 1 (type: int), _col2 (type: string) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col3 (type: bigint) @@ -1269,7 +1269,7 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index 8370bbe..0751849 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -1223,7 +1223,7 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) sort order: +++ Map-reduce partition columns: rand() (type: double) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE @@ -1288,14 +1288,14 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) mode: partials outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: int), _col2 (type: string) + key expressions: _col0 (type: string), 1 (type: int), _col2 (type: string) sort order: +++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: int), _col2 (type: string) + Map-reduce partition columns: _col0 (type: string), 1 (type: int), _col2 (type: string) Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col3 (type: bigint) @@ -1305,7 +1305,7 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: int), KEY._col2 (type: string) + keys: KEY._col0 (type: string), 1 (type: int), KEY._col2 (type: string) mode: final outputColumnNames: _col0, _col1, _col2, _col3 Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out index 346e52f..a94d233 100644 --- a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out +++ b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out @@ -1615,9 +1615,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) + key expressions: 'day' (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string), _col1 (type: string) + Map-reduce partition columns: 'day' (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reducer 2 @@ -1625,7 +1625,7 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: 'day' (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE @@ -1742,9 +1742,9 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) + key expressions: 'day' (type: string), _col1 (type: string) sort order: ++ - Map-reduce partition columns: _col0 (type: string) + Map-reduce partition columns: 'day' (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) Reducer 2 @@ -1752,7 +1752,7 @@ STAGE PLANS: Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) - keys: KEY._col0 (type: string), KEY._col1 (type: string) + keys: 'day' (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out b/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out index 88587e9..f604ed3 100644 --- a/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_null_projection.q.out @@ -119,9 +119,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: void) + key expressions: null (type: void) sort order: + - Map-reduce partition columns: _col0 (type: void) + Map-reduce partition columns: null (type: void) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Map 4 Map Operator Tree: @@ -138,24 +138,28 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: void) + key expressions: null (type: void) sort order: + - Map-reduce partition columns: _col0 (type: void) + Map-reduce partition columns: null (type: void) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reducer 3 Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: void) + keys: null (type: void) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: null (type: void) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Union 2 Vertex: Union 2 http://git-wip-us.apache.org/repos/asf/hive/blob/09e4eb1d/ql/src/test/results/clientpositive/vector_null_projection.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_null_projection.q.out b/ql/src/test/results/clientpositive/vector_null_projection.q.out index c787fd4..941b8e1 100644 --- a/ql/src/test/results/clientpositive/vector_null_projection.q.out +++ b/ql/src/test/results/clientpositive/vector_null_projection.q.out @@ -111,9 +111,9 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: void) + key expressions: null (type: void) sort order: + - Map-reduce partition columns: _col0 (type: void) + Map-reduce partition columns: null (type: void) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE TableScan alias: b @@ -130,23 +130,27 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Output Operator - key expressions: _col0 (type: void) + key expressions: null (type: void) sort order: + - Map-reduce partition columns: _col0 (type: void) + Map-reduce partition columns: null (type: void) Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE Reduce Operator Tree: Group By Operator - keys: KEY._col0 (type: void) + keys: null (type: void) mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false + Select Operator + expressions: null (type: void) + outputColumnNames: _col0 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator
