HIVE-12273: Improve user level explain (Pengcheng Xiong, reviewed by Ashutosh Chauhan, Laljo John Pullokkaran, Eugene Koifman, Prasanth Jayachandran and Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6577f55c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6577f55c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6577f55c Branch: refs/heads/master-fixed Commit: 6577f55cd7f21568994638399f9c31bef578b5cc Parents: d5fdeed Author: Pengcheng Xiong <[email protected]> Authored: Tue Nov 3 10:11:18 2015 -0800 Committer: Pengcheng Xiong <[email protected]> Committed: Tue Nov 3 10:11:18 2015 -0800 ---------------------------------------------------------------------- .../hive/common/jsonexplain/tez/Vertex.java | 9 +- .../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 4 +- .../org/apache/hadoop/hive/ql/plan/MapWork.java | 2 +- .../apache/hadoop/hive/ql/plan/ReduceWork.java | 2 +- .../hadoop/hive/ql/plan/TableScanDesc.java | 7 + .../test/queries/clientpositive/explainuser_3.q | 46 +++- .../clientpositive/llap/constprog_dpp.q.out | 10 +- .../clientpositive/tez/explainuser_1.q.out | 16 ++ .../clientpositive/tez/explainuser_2.q.out | 38 +++ .../clientpositive/tez/explainuser_3.q.out | 230 ++++++++++++++++++- .../tez/vector_aggregate_without_gby.q.out | 4 +- .../tez/vector_auto_smb_mapjoin_14.q.out | 32 +-- .../tez/vectorized_parquet_types.q.out | 2 +- 13 files changed, 363 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java index 67ff8eb..be01b8b 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/tez/Vertex.java @@ -50,6 +50,8 @@ public final class Vertex implements Comparable<Vertex>{ public final List<Vertex> mergeJoinDummyVertexs = new ArrayList<>(); // whether this vertex has multiple reduce operators public boolean hasMultiReduceOp = false; + // execution mode + public String executionMode = ""; public Vertex(String name, JSONObject vertexObject, TezJsonParser tezJsonParser) { super(); @@ -103,6 +105,8 @@ public final class Vertex implements Comparable<Vertex>{ } else { throw new Exception("Merge File Operator does not have a Map Operator Tree"); } + } else if (key.equals("Execution mode:")) { + executionMode = " " + vertexObject.getString(key); } else { throw new Exception("Unsupported operator tree in vertex " + this.name); } @@ -189,9 +193,10 @@ public final class Vertex implements Comparable<Vertex>{ } parser.printSet.add(this); if (type != null) { - printer.println(TezJsonParser.prefixString(indentFlag, "|<-") + this.name + " [" + type + "]"); + printer.println(TezJsonParser.prefixString(indentFlag, "|<-") + this.name + " [" + type + "]" + + this.executionMode); } else if (this.name != null) { - printer.println(TezJsonParser.prefixString(indentFlag) + this.name); + printer.println(TezJsonParser.prefixString(indentFlag) + this.name + this.executionMode); } // print operators if (hasMultiReduceOp && !callingVertex.union) { http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java index e27b89b..4b93e7c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java @@ -334,7 +334,7 @@ public class MapJoinDesc extends JoinDesc implements Serializable { this.bigTablePartSpecToFileMapping = partToFileMapping; } - @Explain(displayName = "BucketMapJoin", explainLevels = { Level.EXTENDED }, displayOnlyOnTrue = true) + @Explain(displayName = "BucketMapJoin", explainLevels = { Level.USER, Level.EXTENDED }, displayOnlyOnTrue = true) public boolean isBucketMapJoin() { return isBucketMapJoin; } @@ -343,7 +343,7 @@ public class MapJoinDesc extends JoinDesc implements Serializable { this.isBucketMapJoin = isBucketMapJoin; } - @Explain(displayName = "HybridGraceHashJoin", displayOnlyOnTrue = true) + @Explain(displayName = "HybridGraceHashJoin", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }, displayOnlyOnTrue = true) public boolean isHybridHashJoin() { return isHybridHashJoin; } http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index 87c15a2..d349934 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -340,7 +340,7 @@ public class MapWork extends BaseWork { } } - @Explain(displayName = "Execution mode") + @Explain(displayName = "Execution mode", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public String getExecutionMode() { if (vectorMode) { if (llapMode) { http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java index 0222c23..8211346 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java @@ -146,7 +146,7 @@ public class ReduceWork extends BaseWork { this.tagToValueDesc = tagToValueDesc; } - @Explain(displayName = "Execution mode") + @Explain(displayName = "Execution mode", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public String getExecutionMode() { if (vectorMode) { if (llapMode) { http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java index 6661ce6..be7139c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java @@ -26,6 +26,8 @@ import java.util.Map; import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; +import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.TableSample; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -135,6 +137,11 @@ public class TableScanDesc extends AbstractOperatorDesc { return alias; } + @Explain(displayName = "ACID table", explainLevels = { Level.USER }, displayOnlyOnTrue = true) + public boolean isAcidTable() { + return SemanticAnalyzer.isAcidTable(this.tableMetadata); + } + @Explain(displayName = "filterExpr") public String getFilterExprString() { StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/test/queries/clientpositive/explainuser_3.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/explainuser_3.q b/ql/src/test/queries/clientpositive/explainuser_3.q index 16237bb..f604d38 100644 --- a/ql/src/test/queries/clientpositive/explainuser_3.q +++ b/ql/src/test/queries/clientpositive/explainuser_3.q @@ -1,5 +1,15 @@ set hive.explain.user=true; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.enforce.bucketing=true; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.vectorized.execution.enabled=true; + +CREATE TABLE acid_vectorized(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true'); +insert into table acid_vectorized select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10; +explain select a, b from acid_vectorized order by a, b; + explain select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol; @@ -112,4 +122,38 @@ set hive.merge.mapredfiles=true; explain insert overwrite table orc_merge5 select userid,string1,subtype,decimal1,ts from orc_merge5 where userid<=13; -drop table orc_merge5; \ No newline at end of file +drop table orc_merge5; + +set hive.auto.convert.join=true; +set hive.auto.convert.join.noconditionaltask=true; +set hive.auto.convert.join.noconditionaltask.size=10000; + +CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE; +CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE; +CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE; + +load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08'); +load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08'); + +load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08'); +load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08'); +load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08'); +load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08'); + +set hive.enforce.bucketing=true; +set hive.enforce.sorting = true; +set hive.optimize.bucketingsorting=false; +insert overwrite table tab_part partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin_part; + +CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE; +insert overwrite table tab partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin; + +set hive.convert.join.bucket.mapjoin.tez = true; +explain +select a.key, a.value, b.value +from tab a join tab_part b on a.key = b.key; + + + http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out b/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out index 0bc964b..72a5d0d 100644 --- a/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out +++ b/ql/src/test/results/clientpositive/llap/constprog_dpp.q.out @@ -50,7 +50,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 4 + Reducer 4 llap File Output Operator [FS_16] compressed:false Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE @@ -60,7 +60,7 @@ Stage-0 | keys:{"0":"id (type: int)","1":"_col0 (type: int)"} | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - |<-Map 6 [SIMPLE_EDGE] + |<-Map 6 [SIMPLE_EDGE] llap | Reduce Output Operator [RS_12] | key expressions:id (type: int) | Map-reduce partition columns:id (type: int) @@ -69,7 +69,7 @@ Stage-0 | TableScan [TS_11] | alias:a | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - |<-Reducer 3 [SIMPLE_EDGE] + |<-Reducer 3 [SIMPLE_EDGE] llap Reduce Output Operator [RS_13] key expressions:_col0 (type: int) Map-reduce partition columns:_col0 (type: int) @@ -82,7 +82,7 @@ Stage-0 | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE |<-Union 2 [SIMPLE_EDGE] - |<-Map 1 [CONTAINS] + |<-Map 1 [CONTAINS] llap | Reduce Output Operator [RS_8] | sort order: | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE @@ -96,7 +96,7 @@ Stage-0 | TableScan [TS_0] | alias:tb2 | Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE - |<-Map 5 [CONTAINS] + |<-Map 5 [CONTAINS] llap Reduce Output Operator [RS_8] sort order: Statistics:Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/test/results/clientpositive/tez/explainuser_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out index ec434f0..ee70033 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out @@ -4683,6 +4683,7 @@ Stage-0 Statistics:Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator [MAPJOIN_25] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"_col0 (type: string)","Map 4":"_col0 (type: string)"} | outputColumnNames:["_col1"] | Statistics:Num rows: 60 Data size: 5160 Basic stats: COMPLETE Column stats: COMPLETE @@ -6418,6 +6419,7 @@ Stage-0 | value expressions:_col0 (type: string) | Map Join Operator [MAPJOIN_28] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 1":"_col0 (type: string)","Map 3":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE @@ -6494,6 +6496,7 @@ Stage-0 | value expressions:_col0 (type: string) | Map Join Operator [MAPJOIN_28] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 1":"_col0 (type: string)","Map 3":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE @@ -6570,6 +6573,7 @@ Stage-0 | value expressions:_col0 (type: string) | Map Join Operator [MAPJOIN_28] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 1":"_col0 (type: string)","Map 3":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col1"] | | Statistics:Num rows: 241 Data size: 42898 Basic stats: COMPLETE Column stats: COMPLETE @@ -6742,6 +6746,7 @@ Stage-0 value expressions:_col5 (type: int) Map Join Operator [MAPJOIN_21] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"p_partkey (type: int)","Map 4":"p_partkey (type: int)"} | outputColumnNames:["_col1","_col2","_col5"] | Statistics:Num rows: 29 Data size: 6467 Basic stats: COMPLETE Column stats: COMPLETE @@ -7014,6 +7019,7 @@ Stage-0 table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} Map Join Operator [MAPJOIN_16] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Reducer 2":"_col0 (type: int)","Map 3":"p_partkey (type: int)"} | outputColumnNames:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"] | Statistics:Num rows: 29 Data size: 17951 Basic stats: COMPLETE Column stats: COMPLETE @@ -7463,6 +7469,7 @@ Stage-0 value expressions:_col5 (type: int), _col7 (type: double) Map Join Operator [MAPJOIN_20] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Reducer 2":"_col0 (type: int)","Map 4":"p_partkey (type: int)"} | outputColumnNames:["_col1","_col2","_col5","_col7"] | Statistics:Num rows: 29 Data size: 6699 Basic stats: COMPLETE Column stats: COMPLETE @@ -8247,6 +8254,7 @@ Stage-0 Statistics:Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_31] | condition map:[{"":"Inner Join 0 to 1"},{"":"Inner Join 1 to 2"},{"":"Inner Join 2 to 3"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"key (type: string)","Map 2":"key (type: string)","Map 3":"key (type: string)","Map 4":"key (type: string)"} | outputColumnNames:["_col0","_col1","_col5","_col6","_col10","_col11","_col15","_col16"] | Statistics:Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE @@ -8327,6 +8335,7 @@ Stage-0 Statistics:Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_31] | condition map:[{"":"Inner Join 0 to 1"},{"":"Inner Join 1 to 2"},{"":"Inner Join 2 to 3"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"key (type: string)","Map 2":"key (type: string)","Map 3":"key (type: string)","Map 4":"key (type: string)"} | outputColumnNames:["_col0","_col1","_col5","_col6","_col10","_col11","_col15","_col16"] | Statistics:Num rows: 3 Data size: 99 Basic stats: COMPLETE Column stats: NONE @@ -8417,6 +8426,7 @@ Stage-0 value expressions:hash(_col0) (type: int), hash(_col1) (type: int), hash(_col5) (type: int) Map Join Operator [MAPJOIN_18] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"UDFToDouble(key) (type: double)","Map 2":"(key + 1) (type: double)"} | outputColumnNames:["_col0","_col1","_col5"] | Statistics:Num rows: 275 Data size: 23925 Basic stats: COMPLETE Column stats: NONE @@ -8558,6 +8568,7 @@ Stage-0 value expressions:hash(_col0) (type: int), hash(_col6) (type: int) Map Join Operator [MAPJOIN_18] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"key (type: string)","Map 4":"val (type: string)"} | outputColumnNames:["_col0","_col6"] | Statistics:Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE @@ -8621,6 +8632,7 @@ Stage-0 value expressions:hash(_col0) (type: int), hash(_col6) (type: int) Map Join Operator [MAPJOIN_18] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"key (type: string)","Map 4":"key (type: string)"} | outputColumnNames:["_col0","_col6"] | Statistics:Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE @@ -8684,6 +8696,7 @@ Stage-0 Statistics:Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_18] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"key (type: string)","Map 4":"key (type: string)"} | Statistics:Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE |<-Map 4 [BROADCAST_EDGE] @@ -8746,6 +8759,7 @@ Stage-0 value expressions:hash(_col0) (type: int), hash(_col1) (type: int), hash(_col5) (type: int) Map Join Operator [MAPJOIN_14] | condition map:[{"":"Left Outer Join0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"UDFToDouble(key) (type: double)","Map 4":"(key + 1) (type: double)"} | outputColumnNames:["_col0","_col1","_col5"] | Statistics:Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE @@ -8804,6 +8818,7 @@ Stage-0 value expressions:hash(_col0) (type: int), hash(_col1) (type: int), hash(_col5) (type: int) Map Join Operator [MAPJOIN_14] | condition map:[{"":"Right Outer Join0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"UDFToDouble(key) (type: double)","Map 2":"(key + 1) (type: double)"} | outputColumnNames:["_col0","_col1","_col5"] | Statistics:Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE @@ -8927,6 +8942,7 @@ Stage-0 value expressions:hash(_col0) (type: int), hash(_col6) (type: int) Map Join Operator [MAPJOIN_14] | condition map:[{"":"Left Outer Join0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"(key + 1) (type: double)","Map 4":"UDFToDouble(key) (type: double)"} | outputColumnNames:["_col0","_col6"] | Statistics:Num rows: 1 Data size: 33 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/test/results/clientpositive/tez/explainuser_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out index 7b361ac..ff055ea 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_2.q.out @@ -1405,6 +1405,7 @@ Stage-0 Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_28] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"_col3 (type: string)","Map 3":"_col0 (type: string)"} | outputColumnNames:["_col0","_col3","_col6"] | Statistics:Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE @@ -1426,6 +1427,7 @@ Stage-0 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE |<-Map Join Operator [MAPJOIN_27] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 1":"_col0 (type: string)","Map 2":"_col1 (type: string)"} | outputColumnNames:["_col0","_col3"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -1566,6 +1568,7 @@ Stage-0 Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_110] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 2":"_col1 (type: string), _col3 (type: string)","Map 3":"_col15 (type: string), _col17 (type: string)"} | outputColumnNames:["_col2","_col3","_col12","_col13","_col20","_col21"] | Statistics:Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE @@ -1578,6 +1581,7 @@ Stage-0 | value expressions:_col2 (type: string) | Map Join Operator [MAPJOIN_104] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 1":"_col0 (type: string)","Map 2":"_col0 (type: string)"} | | outputColumnNames:["_col1","_col2","_col3"] | | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE @@ -1611,6 +1615,7 @@ Stage-0 Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_109] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 3":"_col4 (type: string), _col6 (type: string)","Map 10":"_col2 (type: string), _col4 (type: string)"} | outputColumnNames:["_col2","_col3","_col14","_col15","_col17"] | Statistics:Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE @@ -1623,6 +1628,7 @@ Stage-0 | value expressions:_col3 (type: string), _col5 (type: string) | Map Join Operator [MAPJOIN_108] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 9":"_col0 (type: string)","Map 10":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col3","_col4","_col5"] | | Statistics:Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE @@ -1653,6 +1659,7 @@ Stage-0 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE |<-Map Join Operator [MAPJOIN_107] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 3":"_col3 (type: string)","Map 8":"_col1 (type: string)"} | outputColumnNames:["_col2","_col3","_col4","_col6"] | Statistics:Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE @@ -1673,6 +1680,7 @@ Stage-0 | Statistics:Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE |<-Map Join Operator [MAPJOIN_106] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 3":"_col2 (type: string)","Map 7":"_col0 (type: string)"} | outputColumnNames:["_col2","_col3","_col4","_col6"] | Statistics:Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE @@ -1693,6 +1701,7 @@ Stage-0 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE |<-Map Join Operator [MAPJOIN_105] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 3":"_col1 (type: string)","Map 6":"_col3 (type: string)"} | outputColumnNames:["_col2","_col3","_col4","_col6"] | Statistics:Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE @@ -1780,6 +1789,7 @@ Stage-0 | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_85] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Reducer 11":"_col2 (type: string)","Map 14":"_col0 (type: string)"} | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -1800,6 +1810,7 @@ Stage-0 | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | |<-Map Join Operator [MAPJOIN_84] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Reducer 11":"_col1 (type: string)","Map 13":"_col1 (type: string)"} | | outputColumnNames:["_col1","_col2"] | | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE @@ -1880,6 +1891,7 @@ Stage-0 Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_83] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Reducer 3":"_col2 (type: string)","Map 8":"_col0 (type: string)"} | outputColumnNames:["_col1","_col2"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -1900,6 +1912,7 @@ Stage-0 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE |<-Map Join Operator [MAPJOIN_82] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Reducer 3":"_col1 (type: string)","Map 7":"_col1 (type: string)"} | outputColumnNames:["_col1","_col2"] | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE @@ -2042,6 +2055,7 @@ Stage-0 | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_167] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Reducer 26":"_col2 (type: string)","Map 31":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -2063,6 +2077,7 @@ Stage-0 | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | |<-Map Join Operator [MAPJOIN_166] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Reducer 26":"_col1 (type: string)","Map 30":"_col1 (type: string)"} | | outputColumnNames:["_col2"] | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE @@ -2226,6 +2241,7 @@ Stage-0 | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_165] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Reducer 15":"_col2 (type: string)","Map 19":"_col0 (type: string)"} | | outputColumnNames:["_col2","_col5"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -2247,6 +2263,7 @@ Stage-0 | | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE | |<-Map Join Operator [MAPJOIN_164] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Reducer 15":"_col1 (type: string)","Map 18":"_col1 (type: string)"} | | outputColumnNames:["_col2"] | | Statistics:Num rows: 209 Data size: 2208 Basic stats: COMPLETE Column stats: NONE @@ -2361,6 +2378,7 @@ Stage-0 Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_163] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Reducer 3":"_col2 (type: string)","Map 10":"_col0 (type: string)"} | outputColumnNames:["_col2","_col5"] | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -2382,6 +2400,7 @@ Stage-0 | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE |<-Map Join Operator [MAPJOIN_162] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Reducer 3":"_col1 (type: string)","Map 9":"_col1 (type: string)"} | outputColumnNames:["_col2"] | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE @@ -3058,6 +3077,7 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 17":"_col1 (type: string)","Map 18":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE @@ -3070,6 +3090,7 @@ Stage-0 | | value expressions:_col0 (type: string), _col3 (type: string) | | Map Join Operator [MAPJOIN_119] | | | condition map:[{"":"Inner Join 0 to 1"}] + | | | HybridGraceHashJoin:true | | | keys:{"Map 16":"_col0 (type: string)","Map 17":"_col0 (type: string)"} | | | outputColumnNames:["_col0","_col1","_col3"] | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE @@ -3138,6 +3159,7 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 17":"_col1 (type: string)","Map 19":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE @@ -3161,6 +3183,7 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 17":"_col1 (type: string)","Map 20":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE @@ -3184,6 +3207,7 @@ Stage-0 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_120] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 17":"_col1 (type: string)","Map 21":"_col0 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE @@ -3337,6 +3361,7 @@ Stage-0 | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_115] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 1":"_col0 (type: string)","Map 6":"_col1 (type: string)"} | | outputColumnNames:["_col1"] | | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE @@ -3380,6 +3405,7 @@ Stage-0 Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_115] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Map 5":"_col0 (type: string)","Map 6":"_col1 (type: string)"} | outputColumnNames:["_col1"] | Statistics:Num rows: 289 Data size: 3030 Basic stats: COMPLETE Column stats: NONE @@ -3474,6 +3500,7 @@ Stage-0 | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_164] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 24":"_col1 (type: string)","Reducer 31":"_col1 (type: string)"} | | outputColumnNames:["_col0","_col3"] | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE @@ -3486,6 +3513,7 @@ Stage-0 | | value expressions:_col0 (type: string), _col3 (type: string) | | Map Join Operator [MAPJOIN_163] | | | condition map:[{"":"Inner Join 0 to 1"}] + | | | HybridGraceHashJoin:true | | | keys:{"Map 23":"_col0 (type: string)","Map 24":"_col0 (type: string)"} | | | outputColumnNames:["_col0","_col1","_col3"] | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE @@ -3832,6 +3860,7 @@ Stage-0 Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE Map Join Operator [MAPJOIN_159] | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true | keys:{"Reducer 3":"_col1 (type: string)","Map 10":"_col1 (type: string)"} | outputColumnNames:["_col2"] | Statistics:Num rows: 144 Data size: 1509 Basic stats: COMPLETE Column stats: NONE @@ -3997,6 +4026,7 @@ Stage-5 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_108] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 21":"_col1 (type: string)","Map 16":"_col1 (type: string)"} | | outputColumnNames:["_col0","_col6"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE @@ -4009,6 +4039,7 @@ Stage-5 | | value expressions:_col0 (type: string), _col6 (type: string) | | Map Join Operator [MAPJOIN_105] | | | condition map:[{"":"Inner Join 0 to 1"}] + | | | HybridGraceHashJoin:true | | | keys:{"Map 20":"key (type: string)","Map 21":"key (type: string)"} | | | outputColumnNames:["_col0","_col1","_col6"] | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE @@ -4081,6 +4112,7 @@ Stage-5 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_108] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 21":"_col1 (type: string)","Map 17":"_col1 (type: string)"} | | outputColumnNames:["_col0","_col6"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE @@ -4114,6 +4146,7 @@ Stage-5 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_108] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 21":"_col1 (type: string)","Map 18":"_col1 (type: string)"} | | outputColumnNames:["_col0","_col6"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE @@ -4147,6 +4180,7 @@ Stage-5 | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_108] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 21":"_col1 (type: string)","Map 19":"_col1 (type: string)"} | | outputColumnNames:["_col0","_col6"] | | Statistics:Num rows: 839 Data size: 8873 Basic stats: COMPLETE Column stats: NONE @@ -4299,6 +4333,7 @@ Stage-5 | value expressions:_col0 (type: string), _col6 (type: string) | Map Join Operator [MAPJOIN_103] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 6":"key (type: string)","Map 7":"key (type: string)"} | | outputColumnNames:["_col0","_col1","_col6"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE @@ -4511,6 +4546,7 @@ Stage-5 | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE | Map Join Operator [MAPJOIN_160] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 34":"_col1 (type: string)","Reducer 29":"_col1 (type: string)"} | | outputColumnNames:["_col0","_col6"] | | Statistics:Num rows: 242 Data size: 2565 Basic stats: COMPLETE Column stats: NONE @@ -4523,6 +4559,7 @@ Stage-5 | | value expressions:_col0 (type: string), _col6 (type: string) | | Map Join Operator [MAPJOIN_157] | | | condition map:[{"":"Inner Join 0 to 1"}] + | | | HybridGraceHashJoin:true | | | keys:{"Map 33":"key (type: string)","Map 34":"key (type: string)"} | | | outputColumnNames:["_col0","_col1","_col6"] | | | Statistics:Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE @@ -4843,6 +4880,7 @@ Stage-5 | value expressions:_col0 (type: string), _col6 (type: string) | Map Join Operator [MAPJOIN_155] | | condition map:[{"":"Inner Join 0 to 1"}] + | | HybridGraceHashJoin:true | | keys:{"Map 10":"key (type: string)","Map 11":"key (type: string)"} | | outputColumnNames:["_col0","_col1","_col6"] | | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/test/results/clientpositive/tez/explainuser_3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out index 4f69b3b..880d2ad 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -1,3 +1,55 @@ +PREHOOK: query: CREATE TABLE acid_vectorized(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_vectorized +POSTHOOK: query: CREATE TABLE acid_vectorized(a INT, b STRING) CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_vectorized +PREHOOK: query: insert into table acid_vectorized select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@acid_vectorized +POSTHOOK: query: insert into table acid_vectorized select cint, cstring1 from alltypesorc where cint is not null order by cint limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@acid_vectorized +POSTHOOK: Lineage: acid_vectorized.a SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: acid_vectorized.b SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +PREHOOK: query: explain select a, b from acid_vectorized order by a, b +PREHOOK: type: QUERY +POSTHOOK: query: explain select a, b from acid_vectorized order by a, b +POSTHOOK: type: QUERY +Plan optimized by CBO. + +Vertex dependency in root stage +Reducer 2 <- Map 1 (SIMPLE_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Reducer 2 vectorized + File Output Operator [FS_8] + compressed:false + Statistics:Num rows: 10 Data size: 1704 Basic stats: COMPLETE Column stats: NONE + table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} + Select Operator [OP_7] + | outputColumnNames:["_col0","_col1"] + | Statistics:Num rows: 10 Data size: 1704 Basic stats: COMPLETE Column stats: NONE + |<-Map 1 [SIMPLE_EDGE] vectorized + Reduce Output Operator [RS_6] + key expressions:_col0 (type: int), _col1 (type: string) + sort order:++ + Statistics:Num rows: 10 Data size: 1704 Basic stats: COMPLETE Column stats: NONE + Select Operator [OP_5] + outputColumnNames:["_col0","_col1"] + Statistics:Num rows: 10 Data size: 1704 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_0] + ACID table:true + alias:acid_vectorized + Statistics:Num rows: 10 Data size: 1704 Basic stats: COMPLETE Column stats: NONE + PREHOOK: query: explain select key, value FROM srcpart LATERAL VIEW explode(array(1,2,3)) myTable AS myCol PREHOOK: type: QUERY @@ -448,15 +500,15 @@ Stage-0 Fetch Operator limit:5 Stage-1 - Reducer 2 - File Output Operator [FS_5] + Reducer 2 vectorized + File Output Operator [FS_8] compressed:false Statistics:Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} - Limit [LIM_4] + Limit [LIM_7] Number of rows:5 Statistics:Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE - Select Operator [SEL_3] + Select Operator [OP_6] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE |<-Map 1 [SIMPLE_EDGE] @@ -506,15 +558,15 @@ Stage-3 Stage-8(CONDITIONAL CHILD TASKS: Stage-5, Stage-4, Stage-6) Conditional Operator Stage-1 - Map 1 - File Output Operator [FS_3] + Map 1 vectorized + File Output Operator [FS_10] compressed:false Statistics:Num rows: 306 Data size: 82044 Basic stats: COMPLETE Column stats: NONE table:{"input format:":"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat","output format:":"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat","serde:":"org.apache.hadoop.hive.ql.io.orc.OrcSerde","name:":"default.orc_merge5"} - Select Operator [SEL_2] + Select Operator [OP_9] outputColumnNames:["_col0","_col1","_col2","_col3","_col4"] Statistics:Num rows: 306 Data size: 82044 Basic stats: COMPLETE Column stats: NONE - Filter Operator [FIL_4] + Filter Operator [FIL_8] predicate:(userid <= 13) (type: boolean) Statistics:Num rows: 306 Data size: 82044 Basic stats: COMPLETE Column stats: NONE TableScan [TS_0] @@ -539,3 +591,165 @@ POSTHOOK: query: drop table orc_merge5 POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@orc_merge5 POSTHOOK: Output: default@orc_merge5 +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tab_part +POSTHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab_part +PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin_part +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: insert overwrite table tab_part partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin_part +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin_part +PREHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: Output: default@tab_part@ds=2008-04-08 +POSTHOOK: query: insert overwrite table tab_part partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin_part +POSTHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: Output: default@tab_part@ds=2008-04-08 +POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tab +POSTHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab +PREHOOK: query: insert overwrite table tab partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: Output: default@tab@ds=2008-04-08 +POSTHOOK: query: insert overwrite table tab partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: Output: default@tab@ds=2008-04-08 +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: explain +select a.key, a.value, b.value +from tab a join tab_part b on a.key = b.key +PREHOOK: type: QUERY +POSTHOOK: query: explain +select a.key, a.value, b.value +from tab a join tab_part b on a.key = b.key +POSTHOOK: type: QUERY +Plan not optimized by CBO due to missing statistics. Please check log for more details. + +Vertex dependency in root stage +Map 2 <- Map 1 (CUSTOM_EDGE) + +Stage-0 + Fetch Operator + limit:-1 + Stage-1 + Map 2 + File Output Operator [FS_8] + compressed:false + Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table:{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat","serde:":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"} + Select Operator [SEL_7] + outputColumnNames:["_col0","_col1","_col2"] + Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator [MAPJOIN_13] + | BucketMapJoin:true + | condition map:[{"":"Inner Join 0 to 1"}] + | HybridGraceHashJoin:true + | keys:{"Map 1":"key (type: int)","Map 2":"key (type: int)"} + | outputColumnNames:["_col0","_col1","_col7"] + | Statistics:Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + |<-Map 1 [CUSTOM_EDGE] + | Reduce Output Operator [RS_3] + | key expressions:key (type: int) + | Map-reduce partition columns:key (type: int) + | sort order:+ + | Statistics:Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + | value expressions:value (type: string) + | Filter Operator [FIL_11] + | predicate:key is not null (type: boolean) + | Statistics:Num rows: 121 Data size: 1283 Basic stats: COMPLETE Column stats: NONE + | TableScan [TS_0] + | alias:a + | Statistics:Num rows: 242 Data size: 2566 Basic stats: COMPLETE Column stats: NONE + |<-Filter Operator [FIL_12] + predicate:key is not null (type: boolean) + Statistics:Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan [TS_1] + alias:b + Statistics:Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out b/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out index 9718871..1d84e3b 100644 --- a/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out @@ -46,7 +46,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_7] compressed:false Statistics:Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: NONE @@ -55,7 +55,7 @@ Stage-0 | aggregations:["max(VALUE._col0)","max(VALUE._col1)"] | outputColumnNames:["_col0","_col1"] | Statistics:Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: NONE - |<-Map 1 [SIMPLE_EDGE] + |<-Map 1 [SIMPLE_EDGE] vectorized Reduce Output Operator [RS_4] sort order: Statistics:Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out index 480c4e1..cb6de24 100644 --- a/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out @@ -63,7 +63,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_13] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE @@ -146,7 +146,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 + Reducer 3 vectorized File Output Operator [FS_18] compressed:false Statistics:Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE @@ -155,7 +155,7 @@ Stage-0 | aggregations:["count(VALUE._col0)"] | outputColumnNames:["_col0"] | Statistics:Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 2 [SIMPLE_EDGE] + |<-Reducer 2 [SIMPLE_EDGE] vectorized Reduce Output Operator [RS_15] sort order: Statistics:Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE @@ -289,7 +289,7 @@ Stage-0 | keys:{"0":"_col0 (type: int)","1":"_col0 (type: int)"} | outputColumnNames:["_col0","_col1","_col3"] | Statistics:Num rows: 2 Data size: 224 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 2 [SIMPLE_EDGE] + |<-Reducer 2 [SIMPLE_EDGE] vectorized | Reduce Output Operator [RS_51] | key expressions:_col0 (type: int) | Map-reduce partition columns:_col0 (type: int) @@ -331,7 +331,7 @@ Stage-0 | TableScan [TS_0] | alias:a | Statistics:Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE - |<-Reducer 6 [SIMPLE_EDGE] + |<-Reducer 6 [SIMPLE_EDGE] vectorized Reduce Output Operator [RS_53] key expressions:_col0 (type: int) Map-reduce partition columns:_col0 (type: int) @@ -445,7 +445,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_16] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE @@ -543,7 +543,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_16] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE @@ -662,7 +662,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_20] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE @@ -774,7 +774,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_16] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE @@ -863,7 +863,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 + Reducer 3 vectorized File Output Operator [FS_14] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE @@ -885,7 +885,7 @@ Stage-0 | condition map:[{"":"Inner Join 0 to 1"}] | keys:{"0":"_col0 (type: int)","1":"_col0 (type: int)"} | Statistics:Num rows: 5 Data size: 511 Basic stats: COMPLETE Column stats: NONE - |<-Map 1 [SIMPLE_EDGE] + |<-Map 1 [SIMPLE_EDGE] vectorized | Reduce Output Operator [RS_22] | key expressions:_col0 (type: int) | Map-reduce partition columns:_col0 (type: int) @@ -900,7 +900,7 @@ Stage-0 | TableScan [TS_0] | alias:a | Statistics:Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: NONE - |<-Map 4 [SIMPLE_EDGE] + |<-Map 4 [SIMPLE_EDGE] vectorized Reduce Output Operator [RS_25] key expressions:_col0 (type: int) Map-reduce partition columns:_col0 (type: int) @@ -958,7 +958,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_14] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE @@ -1045,7 +1045,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_21] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE @@ -1161,7 +1161,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_17] compressed:false Statistics:Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE @@ -1448,7 +1448,7 @@ Stage-4 Stage-3 Dependency Collection{} Stage-2 - Reducer 2 + Reducer 2 vectorized File Output Operator [FS_25] compressed:false Statistics:Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: NONE http://git-wip-us.apache.org/repos/asf/hive/blob/6577f55c/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out b/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out index a7ff528..0cb2270 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_parquet_types.q.out @@ -249,7 +249,7 @@ Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 3 + Reducer 3 vectorized File Output Operator [FS_10] compressed:false Statistics:Num rows: 11 Data size: 121 Basic stats: COMPLETE Column stats: NONE
