HIVE-12824 : CBO doesnt get triggered when aggregate function is used within windowing function (Ashutosh Chauhan via Jesus Camacho Rodriguez)
Signed-off-by: Ashutosh Chauhan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/de7810a4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/de7810a4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/de7810a4 Branch: refs/heads/master Commit: de7810a4e17eb22f4dccf32e32f809868ebc3798 Parents: 91d0c83 Author: Ashutosh Chauhan <[email protected]> Authored: Fri Jan 8 17:45:47 2016 -0800 Committer: Ashutosh Chauhan <[email protected]> Committed: Mon Jan 11 14:07:15 2016 -0800 ---------------------------------------------------------------------- .../translator/PlanModifierForASTConv.java | 14 ++ .../test/queries/clientpositive/perf/query51.q | 1 + .../test/queries/clientpositive/perf/query67.q | 1 + .../test/queries/clientpositive/perf/query70.q | 1 + .../test/queries/clientpositive/windowing_gby.q | 1 + .../clientpositive/groupby_resolution.q.out | 4 +- .../results/clientpositive/perf/query51.q.out | 64 ++--- .../results/clientpositive/perf/query67.q.out | 249 ++++++++++--------- .../results/clientpositive/perf/query70.q.out | 73 +++--- .../results/clientpositive/quotedid_basic.q.out | 32 ++- .../spark/groupby_resolution.q.out | 4 +- .../clientpositive/tez/windowing_gby.q.out | 81 +++--- 12 files changed, 280 insertions(+), 245 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java index b77beb8..e2fbb4f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierForASTConv.java @@ -32,10 +32,12 @@ import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.SetOp; import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.core.Window.RexWinAggCall; import org.apache.calcite.rel.rules.MultiJoin; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.util.Pair; import org.slf4j.Logger; @@ -295,6 +297,18 @@ public class PlanModifierForASTConv { validParent = false; } + if (parent instanceof Project) { + for (RexNode child : parent.getChildExps()) { + if (child instanceof RexOver || child instanceof RexWinAggCall) { + // Hive can't handle select rank() over(order by sum(c1)/sum(c2)) from t1 group by c3 + // but can handle select rank() over (order by c4) from + // (select sum(c1)/sum(c2) as c4 from t1 group by c3) t2; + // so introduce a project on top of this gby. + return false; + } + } + } + return validParent; } http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/queries/clientpositive/perf/query51.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/perf/query51.q b/ql/src/test/queries/clientpositive/perf/query51.q index 984280a..b7688cf 100644 --- a/ql/src/test/queries/clientpositive/perf/query51.q +++ b/ql/src/test/queries/clientpositive/perf/query51.q @@ -1,3 +1,4 @@ +set hive.mapred.mode=nonstrict; explain WITH web_v1 as ( select ws_item_sk item_sk, d_date, sum(ws_sales_price), http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/queries/clientpositive/perf/query67.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/perf/query67.q b/ql/src/test/queries/clientpositive/perf/query67.q index 1c03537..56ef907 100644 --- a/ql/src/test/queries/clientpositive/perf/query67.q +++ b/ql/src/test/queries/clientpositive/perf/query67.q @@ -1,3 +1,4 @@ +set hive.mapred.mode=nonstrict; explain select * from (select i_category http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/queries/clientpositive/perf/query70.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/perf/query70.q b/ql/src/test/queries/clientpositive/perf/query70.q index df74bfb..07d68e7 100644 --- a/ql/src/test/queries/clientpositive/perf/query70.q +++ b/ql/src/test/queries/clientpositive/perf/query70.q @@ -1 +1,2 @@ +set hive.mapred.mode=nonstrict; explain select sum(ss_net_profit) as total_sum ,s_state ,s_county ,grouping__id as lochierarchy , rank() over(partition by grouping__id, case when grouping__id == 2 then s_state end order by sum(ss_net_profit)) as rank_within_parent from store_sales ss join date_dim d1 on d1.d_date_sk = ss.ss_sold_date_sk join store s on s.s_store_sk = ss.ss_store_sk where d1.d_month_seq between 1193 and 1193+11 and s.s_state in ( select s_state from (select s_state as s_state, sum(ss_net_profit), rank() over ( partition by s_state order by sum(ss_net_profit) desc) as ranking from store_sales, store, date_dim where d_month_seq between 1193 and 1193+11 and date_dim.d_date_sk = store_sales.ss_sold_date_sk and store.s_store_sk = store_sales.ss_store_sk group by s_state ) tmp1 where ranking <= 5 ) group by s_state,s_county with rollup order by lochierarchy desc ,case when lochierarchy = 0 then s_state end ,rank_within_parent limit 100; http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/queries/clientpositive/windowing_gby.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/windowing_gby.q b/ql/src/test/queries/clientpositive/windowing_gby.q index b948f76..d844f11 100644 --- a/ql/src/test/queries/clientpositive/windowing_gby.q +++ b/ql/src/test/queries/clientpositive/windowing_gby.q @@ -1,3 +1,4 @@ +set hive.mapred.mode=nonstrict; explain select rank() over (order by return_ratio) as return_rank from (select sum(wr.cint)/sum(ws.c_int) as return_ratio http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/results/clientpositive/groupby_resolution.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/groupby_resolution.q.out b/ql/src/test/results/clientpositive/groupby_resolution.q.out index ea40014..9e58b75 100644 --- a/ql/src/test/results/clientpositive/groupby_resolution.q.out +++ b/ql/src/test/results/clientpositive/groupby_resolution.q.out @@ -666,10 +666,10 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: 0 (type: int) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) + value expressions: _col0 (type: string) Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint) + expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE PTF Operator http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/results/clientpositive/perf/query51.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/query51.q.out b/ql/src/test/results/clientpositive/perf/query51.q.out index efd95f2..f57af2c 100644 --- a/ql/src/test/results/clientpositive/perf/query51.q.out +++ b/ql/src/test/results/clientpositive/perf/query51.q.out @@ -100,127 +100,127 @@ Stage-0 limit:100 Stage-1 Reducer 6 - File Output Operator [FS_51] + File Output Operator [FS_53] compressed:false Statistics:Num rows: 100 Data size: 111900 Basic stats: COMPLETE Column stats: NONE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Limit [LIM_50] + Limit [LIM_52] Number of rows:100 Statistics:Num rows: 100 Data size: 111900 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_49] + Select Operator [SEL_51] | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] | Statistics:Num rows: 7365 Data size: 8241815 Basic stats: COMPLETE Column stats: NONE |<-Reducer 5 [SIMPLE_EDGE] - Reduce Output Operator [RS_48] + Reduce Output Operator [RS_50] key expressions:_col0 (type: int), _col1 (type: string) sort order:++ Statistics:Num rows: 7365 Data size: 8241815 Basic stats: COMPLETE Column stats: NONE value expressions:_col2 (type: decimal(27,2)), _col3 (type: decimal(27,2)), _col4 (type: decimal(27,2)), _col5 (type: decimal(27,2)) - Select Operator [SEL_44] + Select Operator [SEL_46] outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] Statistics:Num rows: 7365 Data size: 8241815 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_56] + Filter Operator [FIL_58] predicate:(max_window_0 > max_window_1) (type: boolean) Statistics:Num rows: 7365 Data size: 8241815 Basic stats: COMPLETE Column stats: NONE - PTF Operator [PTF_43] + PTF Operator [PTF_45] Function definitions:[{"Input definition":{"type:":"WINDOWING"}},{"name:":"windowingtablefunction","order by:":"CASE WHEN (_col1 is not null) THEN (_col1) ELSE (_col4) END","partition by:":"CASE WHEN (_col0 is not null) THEN (_col0) ELSE (_col3) END"}] Statistics:Num rows: 22096 Data size: 24726566 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_42] + Select Operator [SEL_44] | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] | Statistics:Num rows: 22096 Data size: 24726566 Basic stats: COMPLETE Column stats: NONE |<-Reducer 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_41] + Reduce Output Operator [RS_43] key expressions:CASE WHEN (_col0 is not null) THEN (_col0) ELSE (_col3) END (type: int), CASE WHEN (_col1 is not null) THEN (_col1) ELSE (_col4) END (type: string) Map-reduce partition columns:CASE WHEN (_col0 is not null) THEN (_col0) ELSE (_col3) END (type: int) sort order:++ Statistics:Num rows: 22096 Data size: 24726566 Basic stats: COMPLETE Column stats: NONE value expressions:_col0 (type: int), _col1 (type: string), _col2 (type: decimal(27,2)), _col3 (type: int), _col4 (type: string), _col5 (type: decimal(27,2)) - Merge Join Operator [MERGEJOIN_65] + Merge Join Operator [MERGEJOIN_67] | condition map:[{"":"Outer Join 0 to 1"}] | keys:{"0":"_col0 (type: int), _col1 (type: string)","1":"_col0 (type: int), _col1 (type: string)"} | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5"] | Statistics:Num rows: 22096 Data size: 24726566 Basic stats: COMPLETE Column stats: NONE |<-Reducer 10 [SIMPLE_EDGE] - | Reduce Output Operator [RS_39] + | Reduce Output Operator [RS_41] | key expressions:_col0 (type: int), _col1 (type: string) | Map-reduce partition columns:_col0 (type: int), _col1 (type: string) | sort order:++ | Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE | value expressions:_col2 (type: decimal(27,2)) - | Select Operator [SEL_35] + | Select Operator [SEL_37] | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE - | PTF Operator [PTF_34] + | PTF Operator [PTF_36] | Function definitions:[{"Input definition":{"type:":"WINDOWING"}},{"name:":"windowingtablefunction","order by:":"_col1","partition by:":"_col0"}] | Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE - | Group By Operator [GBY_31] + | Group By Operator [GBY_32] | | aggregations:["sum(VALUE._col0)"] | | keys:KEY._col0 (type: int), KEY._col1 (type: string) | | outputColumnNames:["_col0","_col1","_col2"] | | Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE | |<-Reducer 9 [SIMPLE_EDGE] - | Reduce Output Operator [RS_30] + | Reduce Output Operator [RS_31] | key expressions:_col0 (type: int), _col1 (type: string) | Map-reduce partition columns:_col0 (type: int) | sort order:++ | Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE | value expressions:_col2 (type: decimal(17,2)) - | Group By Operator [GBY_29] + | Group By Operator [GBY_30] | aggregations:["sum(_col2)"] | keys:_col1 (type: int), _col4 (type: string) | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE - | Select Operator [SEL_28] + | Select Operator [SEL_29] | outputColumnNames:["_col1","_col4","_col2"] | Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE - | Merge Join Operator [MERGEJOIN_64] + | Merge Join Operator [MERGEJOIN_66] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"0":"_col0 (type: int)","1":"_col0 (type: int)"} | | outputColumnNames:["_col1","_col2","_col4"] | | Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE | |<-Map 11 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_26] + | | Reduce Output Operator [RS_27] | | key expressions:_col0 (type: int) | | Map-reduce partition columns:_col0 (type: int) | | sort order:+ | | Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE | | value expressions:_col1 (type: string) - | | Select Operator [SEL_24] + | | Select Operator [SEL_25] | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - | | Filter Operator [FIL_60] + | | Filter Operator [FIL_62] | | predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) (type: boolean) | | Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - | | TableScan [TS_22] + | | TableScan [TS_23] | | alias:date_dim | | Statistics:Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE | |<-Map 8 [SIMPLE_EDGE] - | Reduce Output Operator [RS_25] + | Reduce Output Operator [RS_26] | key expressions:_col0 (type: int) | Map-reduce partition columns:_col0 (type: int) | sort order:+ | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE | value expressions:_col1 (type: int), _col2 (type: decimal(7,2)) - | Select Operator [SEL_21] + | Select Operator [SEL_22] | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - | Filter Operator [FIL_59] + | Filter Operator [FIL_61] | predicate:(ss_item_sk is not null and ss_sold_date_sk is not null) (type: boolean) | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - | TableScan [TS_19] + | TableScan [TS_20] | alias:store_sales | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE |<-Reducer 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_38] + Reduce Output Operator [RS_40] key expressions:_col0 (type: int), _col1 (type: string) Map-reduce partition columns:_col0 (type: int), _col1 (type: string) sort order:++ Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE value expressions:_col2 (type: decimal(27,2)) - Select Operator [SEL_16] + Select Operator [SEL_17] outputColumnNames:["_col0","_col1","_col2"] Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE - PTF Operator [PTF_15] + PTF Operator [PTF_16] Function definitions:[{"Input definition":{"type:":"WINDOWING"}},{"name:":"windowingtablefunction","order by:":"_col1","partition by:":"_col0"}] Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE Group By Operator [GBY_12] @@ -243,7 +243,7 @@ Stage-0 Select Operator [SEL_9] outputColumnNames:["_col1","_col4","_col2"] Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_63] + Merge Join Operator [MERGEJOIN_65] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"0":"_col0 (type: int)","1":"_col0 (type: int)"} | outputColumnNames:["_col1","_col2","_col4"] @@ -258,7 +258,7 @@ Stage-0 | Select Operator [SEL_2] | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - | Filter Operator [FIL_57] + | Filter Operator [FIL_59] | predicate:(ws_item_sk is not null and ws_sold_date_sk is not null) (type: boolean) | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE | TableScan [TS_0] @@ -274,7 +274,7 @@ Stage-0 Select Operator [SEL_5] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_58] + Filter Operator [FIL_60] predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) (type: boolean) Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE TableScan [TS_3] http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/results/clientpositive/perf/query67.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/query67.q.out b/ql/src/test/results/clientpositive/perf/query67.q.out index 0a61d0f..dda5347 100644 --- a/ql/src/test/results/clientpositive/perf/query67.q.out +++ b/ql/src/test/results/clientpositive/perf/query67.q.out @@ -99,151 +99,154 @@ Stage-0 limit:100 Stage-1 Reducer 7 - File Output Operator [FS_36] + File Output Operator [FS_37] compressed:false Statistics:Num rows: 100 Data size: 143600 Basic stats: COMPLETE Column stats: NONE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Limit [LIM_35] + Limit [LIM_36] Number of rows:100 Statistics:Num rows: 100 Data size: 143600 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_34] + Select Operator [SEL_35] | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] | Statistics:Num rows: 762300 Data size: 1094874777 Basic stats: COMPLETE Column stats: NONE |<-Reducer 6 [SIMPLE_EDGE] - Reduce Output Operator [RS_33] + Reduce Output Operator [RS_34] key expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: decimal(28,2)), _col9 (type: int) sort order:++++++++++ Statistics:Num rows: 762300 Data size: 1094874777 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_29] + Select Operator [SEL_30] outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] Statistics:Num rows: 762300 Data size: 1094874777 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_46] + Filter Operator [FIL_47] predicate:(rank_window_0 <= 100) (type: boolean) Statistics:Num rows: 762300 Data size: 1094874777 Basic stats: COMPLETE Column stats: NONE - PTF Operator [PTF_28] - Function definitions:[{"Input definition":{"type:":"WINDOWING"}},{"name:":"windowingtablefunction","order by:":"_col9(DESC)","partition by:":"_col0"}] + PTF Operator [PTF_29] + Function definitions:[{"Input definition":{"type:":"WINDOWING"}},{"name:":"windowingtablefunction","order by:":"_col16(DESC)","partition by:":"_col0"}] Statistics:Num rows: 2286900 Data size: 3284624331 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_27] - | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9"] + Select Operator [SEL_28] + | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col16"] | Statistics:Num rows: 2286900 Data size: 3284624331 Basic stats: COMPLETE Column stats: NONE |<-Reducer 5 [SIMPLE_EDGE] - Reduce Output Operator [RS_26] - key expressions:_col0 (type: string), _col9 (type: decimal(28,2)) + Reduce Output Operator [RS_27] + key expressions:_col0 (type: string), _col16 (type: decimal(28,2)) Map-reduce partition columns:_col0 (type: string) sort order:+- Statistics:Num rows: 2286900 Data size: 3284624331 Basic stats: COMPLETE Column stats: NONE - value expressions:_col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col9 (type: decimal(28,2)) - Group By Operator [GBY_25] - | aggregations:["sum(VALUE._col0)"] - | keys:KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._col5 (type: int), KEY._col6 (type: int), KEY._col7 (type: string), KEY._col8 (type: string) - | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9"] - | Statistics:Num rows: 2286900 Data size: 3284624331 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_24] - key expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: string) - Map-reduce partition columns:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: string) - sort order:+++++++++ - Statistics:Num rows: 4573800 Data size: 6569248662 Basic stats: COMPLETE Column stats: NONE - value expressions:_col9 (type: decimal(28,2)) - Group By Operator [GBY_23] - aggregations:["sum(_col8)"] - keys:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), '0' (type: string) - outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] + value expressions:_col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string) + Select Operator [SEL_26] + outputColumnNames:["_col0","_col1","_col16","_col2","_col3","_col4","_col5","_col6","_col7"] + Statistics:Num rows: 2286900 Data size: 3284624331 Basic stats: COMPLETE Column stats: NONE + Group By Operator [GBY_25] + | aggregations:["sum(VALUE._col0)"] + | keys:KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string), KEY._col3 (type: string), KEY._col4 (type: int), KEY._col5 (type: int), KEY._col6 (type: int), KEY._col7 (type: string), KEY._col8 (type: string) + | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9"] + | Statistics:Num rows: 2286900 Data size: 3284624331 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 4 [SIMPLE_EDGE] + Reduce Output Operator [RS_24] + key expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: string) + Map-reduce partition columns:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), _col8 (type: string) + sort order:+++++++++ Statistics:Num rows: 4573800 Data size: 6569248662 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_21] - outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] - Statistics:Num rows: 508200 Data size: 729916518 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_53] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"0":"_col1 (type: int)","1":"_col0 (type: int)"} - | outputColumnNames:["_col3","_col4","_col7","_col8","_col9","_col11","_col13","_col14","_col15","_col16"] - | Statistics:Num rows: 508200 Data size: 729916518 Basic stats: COMPLETE Column stats: NONE - |<-Map 10 [SIMPLE_EDGE] - | Reduce Output Operator [RS_19] - | key expressions:_col0 (type: int) - | Map-reduce partition columns:_col0 (type: int) - | sort order:+ - | Statistics:Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string) - | Select Operator [SEL_11] - | outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] - | Statistics:Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_50] - | predicate:i_item_sk is not null (type: boolean) - | Statistics:Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_9] - | alias:item - | Statistics:Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_18] - key expressions:_col1 (type: int) - Map-reduce partition columns:_col1 (type: int) - sort order:+ - Statistics:Num rows: 44193 Data size: 49453132 Basic stats: COMPLETE Column stats: NONE - value expressions:_col3 (type: int), _col4 (type: decimal(7,2)), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col11 (type: string) - Merge Join Operator [MERGEJOIN_52] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"0":"_col2 (type: int)","1":"_col0 (type: int)"} - | outputColumnNames:["_col1","_col3","_col4","_col7","_col8","_col9","_col11"] - | Statistics:Num rows: 44193 Data size: 49453132 Basic stats: COMPLETE Column stats: NONE - |<-Map 9 [SIMPLE_EDGE] - | Reduce Output Operator [RS_16] - | key expressions:_col0 (type: int) - | Map-reduce partition columns:_col0 (type: int) - | sort order:+ - | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - | value expressions:_col1 (type: string) - | Select Operator [SEL_8] - | outputColumnNames:["_col0","_col1"] - | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_49] - | predicate:s_store_sk is not null (type: boolean) - | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - | TableScan [TS_6] - | alias:store - | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 2 [SIMPLE_EDGE] - Reduce Output Operator [RS_15] - key expressions:_col2 (type: int) - Map-reduce partition columns:_col2 (type: int) - sort order:+ - Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE - value expressions:_col1 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col7 (type: int), _col8 (type: int), _col9 (type: int) - Merge Join Operator [MERGEJOIN_51] - | condition map:[{"":"Inner Join 0 to 1"}] - | keys:{"0":"_col0 (type: int)","1":"_col0 (type: int)"} - | outputColumnNames:["_col1","_col2","_col3","_col4","_col7","_col8","_col9"] - | Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE - |<-Map 1 [SIMPLE_EDGE] - | Reduce Output Operator [RS_12] - | key expressions:_col0 (type: int) - | Map-reduce partition columns:_col0 (type: int) - | sort order:+ - | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - | value expressions:_col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)) - | Select Operator [SEL_2] - | outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] - | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - | Filter Operator [FIL_47] - | predicate:((ss_sold_date_sk is not null and ss_store_sk is not null) and ss_item_sk is not null) (type: boolean) - | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - | TableScan [TS_0] - | alias:store_sales - | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - |<-Map 8 [SIMPLE_EDGE] - Reduce Output Operator [RS_13] - key expressions:_col0 (type: int) - Map-reduce partition columns:_col0 (type: int) - sort order:+ - Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - value expressions:_col2 (type: int), _col3 (type: int), _col4 (type: int) - Select Operator [SEL_5] - outputColumnNames:["_col0","_col2","_col3","_col4"] + value expressions:_col9 (type: decimal(28,2)) + Group By Operator [GBY_23] + aggregations:["sum(_col8)"] + keys:_col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: int), _col5 (type: int), _col6 (type: int), _col7 (type: string), '0' (type: string) + outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] + Statistics:Num rows: 4573800 Data size: 6569248662 Basic stats: COMPLETE Column stats: NONE + Select Operator [SEL_21] + outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] + Statistics:Num rows: 508200 Data size: 729916518 Basic stats: COMPLETE Column stats: NONE + Merge Join Operator [MERGEJOIN_54] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"0":"_col1 (type: int)","1":"_col0 (type: int)"} + | outputColumnNames:["_col3","_col4","_col7","_col8","_col9","_col11","_col13","_col14","_col15","_col16"] + | Statistics:Num rows: 508200 Data size: 729916518 Basic stats: COMPLETE Column stats: NONE + |<-Map 10 [SIMPLE_EDGE] + | Reduce Output Operator [RS_19] + | key expressions:_col0 (type: int) + | Map-reduce partition columns:_col0 (type: int) + | sort order:+ + | Statistics:Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col1 (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string) + | Select Operator [SEL_11] + | outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] + | Statistics:Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_51] + | predicate:i_item_sk is not null (type: boolean) + | Statistics:Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_9] + | alias:item + | Statistics:Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 3 [SIMPLE_EDGE] + Reduce Output Operator [RS_18] + key expressions:_col1 (type: int) + Map-reduce partition columns:_col1 (type: int) + sort order:+ + Statistics:Num rows: 44193 Data size: 49453132 Basic stats: COMPLETE Column stats: NONE + value expressions:_col3 (type: int), _col4 (type: decimal(7,2)), _col7 (type: int), _col8 (type: int), _col9 (type: int), _col11 (type: string) + Merge Join Operator [MERGEJOIN_53] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"0":"_col2 (type: int)","1":"_col0 (type: int)"} + | outputColumnNames:["_col1","_col3","_col4","_col7","_col8","_col9","_col11"] + | Statistics:Num rows: 44193 Data size: 49453132 Basic stats: COMPLETE Column stats: NONE + |<-Map 9 [SIMPLE_EDGE] + | Reduce Output Operator [RS_16] + | key expressions:_col0 (type: int) + | Map-reduce partition columns:_col0 (type: int) + | sort order:+ + | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + | value expressions:_col1 (type: string) + | Select Operator [SEL_8] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + | Filter Operator [FIL_50] + | predicate:s_store_sk is not null (type: boolean) + | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_6] + | alias:store + | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + |<-Reducer 2 [SIMPLE_EDGE] + Reduce Output Operator [RS_15] + key expressions:_col2 (type: int) + Map-reduce partition columns:_col2 (type: int) + sort order:+ + Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE + value expressions:_col1 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)), _col7 (type: int), _col8 (type: int), _col9 (type: int) + Merge Join Operator [MERGEJOIN_52] + | condition map:[{"":"Inner Join 0 to 1"}] + | keys:{"0":"_col0 (type: int)","1":"_col0 (type: int)"} + | outputColumnNames:["_col1","_col2","_col3","_col4","_col7","_col8","_col9"] + | Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE + |<-Map 1 [SIMPLE_EDGE] + | Reduce Output Operator [RS_12] + | key expressions:_col0 (type: int) + | Map-reduce partition columns:_col0 (type: int) + | sort order:+ + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + | value expressions:_col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)) + | Select Operator [SEL_2] + | outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + | Filter Operator [FIL_48] + | predicate:((ss_sold_date_sk is not null and ss_store_sk is not null) and ss_item_sk is not null) (type: boolean) + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + | TableScan [TS_0] + | alias:store_sales + | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + |<-Map 8 [SIMPLE_EDGE] + Reduce Output Operator [RS_13] + key expressions:_col0 (type: int) + Map-reduce partition columns:_col0 (type: int) + sort order:+ Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_48] - predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) (type: boolean) + value expressions:_col2 (type: int), _col3 (type: int), _col4 (type: int) + Select Operator [SEL_5] + outputColumnNames:["_col0","_col2","_col3","_col4"] Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - TableScan [TS_3] - alias:date_dim - Statistics:Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE + Filter Operator [FIL_49] + predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) (type: boolean) + Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_3] + alias:date_dim + Statistics:Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/results/clientpositive/perf/query70.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/query70.q.out b/ql/src/test/results/clientpositive/perf/query70.q.out index d8fd350..d13165d 100644 --- a/ql/src/test/results/clientpositive/perf/query70.q.out +++ b/ql/src/test/results/clientpositive/perf/query70.q.out @@ -21,80 +21,80 @@ Stage-0 limit:100 Stage-1 Reducer 6 - File Output Operator [FS_61] + File Output Operator [FS_62] compressed:false Statistics:Num rows: 100 Data size: 111900 Basic stats: COMPLETE Column stats: NONE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Limit [LIM_60] + Limit [LIM_61] Number of rows:100 Statistics:Num rows: 100 Data size: 111900 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_59] + Select Operator [SEL_60] | outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] | Statistics:Num rows: 66289 Data size: 74179138 Basic stats: COMPLETE Column stats: NONE |<-Reducer 5 [SIMPLE_EDGE] - Reduce Output Operator [RS_58] + Reduce Output Operator [RS_59] key expressions:_col3 (type: string), CASE WHEN ((_col3 = 0)) THEN (_col1) END (type: string), _col4 (type: int) sort order:-++ Statistics:Num rows: 66289 Data size: 74179138 Basic stats: COMPLETE Column stats: NONE value expressions:_col0 (type: decimal(17,2)), _col1 (type: string), _col2 (type: string) - Select Operator [SEL_56] + Select Operator [SEL_57] outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] Statistics:Num rows: 66289 Data size: 74179138 Basic stats: COMPLETE Column stats: NONE - PTF Operator [PTF_55] + PTF Operator [PTF_56] Function definitions:[{"Input definition":{"type:":"WINDOWING"}},{"name:":"windowingtablefunction","order by:":"_col4","partition by:":"_col5, CASE WHEN ((_col5 = 2)) THEN (_col0) END"}] Statistics:Num rows: 66289 Data size: 74179138 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_54] + Select Operator [SEL_55] | outputColumnNames:["_col0","_col1","_col4","_col5"] | Statistics:Num rows: 66289 Data size: 74179138 Basic stats: COMPLETE Column stats: NONE |<-Reducer 4 [SIMPLE_EDGE] - Reduce Output Operator [RS_53] + Reduce Output Operator [RS_54] key expressions:_col5 (type: string), CASE WHEN ((_col5 = 2)) THEN (_col0) END (type: string), _col4 (type: decimal(17,2)) Map-reduce partition columns:_col5 (type: string), CASE WHEN ((_col5 = 2)) THEN (_col0) END (type: string) sort order:+++ Statistics:Num rows: 66289 Data size: 74179138 Basic stats: COMPLETE Column stats: NONE value expressions:_col0 (type: string), _col1 (type: string) - Select Operator [SEL_52] + Select Operator [SEL_53] outputColumnNames:["_col0","_col1","_col4","_col5"] Statistics:Num rows: 66289 Data size: 74179138 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_51] + Group By Operator [GBY_52] | aggregations:["sum(VALUE._col0)"] | keys:KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string) | outputColumnNames:["_col0","_col1","_col2","_col3"] | Statistics:Num rows: 66289 Data size: 74179138 Basic stats: COMPLETE Column stats: NONE |<-Reducer 3 [SIMPLE_EDGE] - Reduce Output Operator [RS_50] + Reduce Output Operator [RS_51] key expressions:_col0 (type: string), _col1 (type: string), _col2 (type: string) Map-reduce partition columns:_col0 (type: string), _col1 (type: string), _col2 (type: string) sort order:+++ Statistics:Num rows: 132579 Data size: 148359396 Basic stats: COMPLETE Column stats: NONE value expressions:_col3 (type: decimal(17,2)) - Group By Operator [GBY_49] + Group By Operator [GBY_50] aggregations:["sum(_col2)"] keys:_col6 (type: string), _col7 (type: string), '0' (type: string) outputColumnNames:["_col0","_col1","_col2","_col3"] Statistics:Num rows: 132579 Data size: 148359396 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_48] + Select Operator [SEL_49] outputColumnNames:["_col6","_col7","_col2"] Statistics:Num rows: 44193 Data size: 49453132 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_90] + Merge Join Operator [MERGEJOIN_91] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"0":"_col1 (type: int)","1":"_col0 (type: int)"} | outputColumnNames:["_col2","_col6","_col7"] | Statistics:Num rows: 44193 Data size: 49453132 Basic stats: COMPLETE Column stats: NONE |<-Reducer 2 [SIMPLE_EDGE] - | Reduce Output Operator [RS_45] + | Reduce Output Operator [RS_46] | key expressions:_col1 (type: int) | Map-reduce partition columns:_col1 (type: int) | sort order:+ | Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE | value expressions:_col2 (type: decimal(7,2)) - | Merge Join Operator [MERGEJOIN_86] + | Merge Join Operator [MERGEJOIN_87] | | condition map:[{"":"Inner Join 0 to 1"}] | | keys:{"0":"_col0 (type: int)","1":"_col0 (type: int)"} | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE | |<-Map 1 [SIMPLE_EDGE] - | | Reduce Output Operator [RS_42] + | | Reduce Output Operator [RS_43] | | key expressions:_col0 (type: int) | | Map-reduce partition columns:_col0 (type: int) | | sort order:+ @@ -103,14 +103,14 @@ Stage-0 | | Select Operator [SEL_2] | | outputColumnNames:["_col0","_col1","_col2"] | | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - | | Filter Operator [FIL_79] + | | Filter Operator [FIL_80] | | predicate:(ss_sold_date_sk is not null and ss_store_sk is not null) (type: boolean) | | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE | | TableScan [TS_0] | | alias:ss | | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE | |<-Map 7 [SIMPLE_EDGE] - | Reduce Output Operator [RS_43] + | Reduce Output Operator [RS_44] | key expressions:_col0 (type: int) | Map-reduce partition columns:_col0 (type: int) | sort order:+ @@ -118,26 +118,26 @@ Stage-0 | Select Operator [SEL_5] | outputColumnNames:["_col0"] | Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_80] + | Filter Operator [FIL_81] | predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) (type: boolean) | Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE | TableScan [TS_3] | alias:d1 | Statistics:Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE |<-Reducer 9 [SIMPLE_EDGE] - Reduce Output Operator [RS_46] + Reduce Output Operator [RS_47] key expressions:_col0 (type: int) Map-reduce partition columns:_col0 (type: int) sort order:+ Statistics:Num rows: 7365 Data size: 8242187 Basic stats: COMPLETE Column stats: NONE value expressions:_col1 (type: string), _col2 (type: string) - Merge Join Operator [MERGEJOIN_89] + Merge Join Operator [MERGEJOIN_90] | condition map:[{"":"Left Semi Join 0 to 1"}] | keys:{"0":"_col2 (type: string)","1":"_col0 (type: string)"} | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 7365 Data size: 8242187 Basic stats: COMPLETE Column stats: NONE |<-Map 8 [SIMPLE_EDGE] - | Reduce Output Operator [RS_38] + | Reduce Output Operator [RS_39] | key expressions:_col2 (type: string) | Map-reduce partition columns:_col2 (type: string) | sort order:+ @@ -146,41 +146,40 @@ Stage-0 | Select Operator [SEL_8] | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_81] + | Filter Operator [FIL_82] | predicate:(s_store_sk is not null and s_state is not null) (type: boolean) | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE | TableScan [TS_6] | alias:s | Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE |<-Reducer 14 [SIMPLE_EDGE] - Reduce Output Operator [RS_39] + Reduce Output Operator [RS_40] key expressions:_col0 (type: string) Map-reduce partition columns:_col0 (type: string) sort order:+ Statistics:Num rows: 6696 Data size: 7492898 Basic stats: COMPLETE Column stats: NONE - Group By Operator [GBY_37] + Group By Operator [GBY_38] keys:_col0 (type: string) outputColumnNames:["_col0"] Statistics:Num rows: 6696 Data size: 7492898 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_31] + Select Operator [SEL_32] outputColumnNames:["_col0"] Statistics:Num rows: 6696 Data size: 7492898 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_82] + Filter Operator [FIL_83] predicate:((rank_window_0 <= 5) and _col0 is not null) (type: boolean) Statistics:Num rows: 6696 Data size: 7492898 Basic stats: COMPLETE Column stats: NONE - PTF Operator [PTF_30] + PTF Operator [PTF_31] Function definitions:[{"Input definition":{"type:":"WINDOWING"}},{"name:":"windowingtablefunction","order by:":"_col1(DESC)","partition by:":"_col0"}] Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_29] + Select Operator [SEL_30] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE |<-Reducer 13 [SIMPLE_EDGE] - Reduce Output Operator [RS_28] + Reduce Output Operator [RS_29] key expressions:_col0 (type: string), _col1 (type: decimal(17,2)) Map-reduce partition columns:_col0 (type: string) sort order:+- Statistics:Num rows: 20088 Data size: 22478696 Basic stats: COMPLETE Column stats: NONE - value expressions:_col1 (type: decimal(17,2)) Group By Operator [GBY_27] | aggregations:["sum(VALUE._col0)"] | keys:KEY._col0 (type: string) @@ -201,7 +200,7 @@ Stage-0 Select Operator [SEL_24] outputColumnNames:["_col4","_col2"] Statistics:Num rows: 40176 Data size: 44957392 Basic stats: COMPLETE Column stats: NONE - Merge Join Operator [MERGEJOIN_88] + Merge Join Operator [MERGEJOIN_89] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"0":"_col0 (type: int)","1":"_col0 (type: int)"} | outputColumnNames:["_col2","_col4"] @@ -215,7 +214,7 @@ Stage-0 | Select Operator [SEL_17] | outputColumnNames:["_col0"] | Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE - | Filter Operator [FIL_85] + | Filter Operator [FIL_86] | predicate:(d_month_seq BETWEEN 1193 AND 1204 and d_date_sk is not null) (type: boolean) | Statistics:Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE | TableScan [TS_15] @@ -228,7 +227,7 @@ Stage-0 sort order:+ Statistics:Num rows: 1874 Data size: 3581903 Basic stats: COMPLETE Column stats: NONE value expressions:_col2 (type: decimal(7,2)), _col4 (type: string) - Merge Join Operator [MERGEJOIN_87] + Merge Join Operator [MERGEJOIN_88] | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"0":"_col1 (type: int)","1":"_col0 (type: int)"} | outputColumnNames:["_col0","_col2","_col4"] @@ -243,7 +242,7 @@ Stage-0 | Select Operator [SEL_11] | outputColumnNames:["_col0","_col1","_col2"] | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - | Filter Operator [FIL_83] + | Filter Operator [FIL_84] | predicate:(ss_store_sk is not null and ss_sold_date_sk is not null) (type: boolean) | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE | TableScan [TS_9] @@ -259,7 +258,7 @@ Stage-0 Select Operator [SEL_14] outputColumnNames:["_col0","_col1"] Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_84] + Filter Operator [FIL_85] predicate:s_store_sk is not null (type: boolean) Statistics:Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE TableScan [TS_12] http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/results/clientpositive/quotedid_basic.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/quotedid_basic.q.out b/ql/src/test/results/clientpositive/quotedid_basic.q.out index 29736af..5acc4a6 100644 --- a/ql/src/test/results/clientpositive/quotedid_basic.q.out +++ b/ql/src/test/results/clientpositive/quotedid_basic.q.out @@ -179,12 +179,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce @@ -283,12 +287,16 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Select Operator + expressions: _col0 (type: string), _col1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe Stage: Stage-2 Map Reduce http://git-wip-us.apache.org/repos/asf/hive/blob/de7810a4/ql/src/test/results/clientpositive/spark/groupby_resolution.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/groupby_resolution.q.out b/ql/src/test/results/clientpositive/spark/groupby_resolution.q.out index cb2c9bd..cef5b23 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_resolution.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_resolution.q.out @@ -659,11 +659,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: 0 (type: int) Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: bigint) + value expressions: _col0 (type: string) Reducer 4 Reduce Operator Tree: Select Operator - expressions: VALUE._col0 (type: string), VALUE._col1 (type: bigint) + expressions: VALUE._col0 (type: string), KEY.reducesinkkey1 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE PTF Operator
