[2/3] hive git commit: HIVE-18769: Vectorization: Disable vectorization of key-less outer joins (Matt McCline, reviewed by Gopal Vijayaraghavan)

2018-02-22 Thread mmccline 
http://git-wip-us.apache.org/repos/asf/hive/blob/1593c10d/ql/src/test/results/clientpositive/vector_outer_join3.q.out
--
diff --git a/ql/src/test/results/clientpositive/vector_outer_join3.q.out 
b/ql/src/test/results/clientpositive/vector_outer_join3.q.out
index 1ab03fb..57ce681 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join3.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join3.q.out
@@ -244,7 +244,7 @@ left outer join small_alltypesorc_a hd
   on hd.cstring1 = c.cstring1
 ) t1
 POSTHOOK: type: QUERY
-{"PLAN 
VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled
 IS true"]},"STAGE DEPENDENCIES":{"Stage-8":{"ROOT 
STAGE":"TRUE"},"Stage-3":{"DEPENDENT STAGES":"Stage-8"},"Stage-0":{"DEPENDENT 
STAGES":"Stage-3"}},"STAGE PLANS":{"Stage-8":{"Map Reduce Local Work":{"Alias 
-> Map Local Tables:":{"$hdt$_1:cd":{"Fetch 
Operator":{"limit:":"-1"}},"$hdt$_2:hd":{"Fetch 
Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator 
Tree:":{"$hdt$_1:cd":{"TableScan":{"alias:":"cd","columns:":["cint"],"database:":"default","Statistics:":"Num
 rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: 
NONE","table:":"small_alltypesorc_a","isTempTable:":"false","OperatorId:":"TS_2","children":{"Select
 Operator":{"expressions:":"cint (type: 
int)","columnExprMap:":{"_col0":"cint"},"outputColumnNames:":["_col0"],"Statistics:":"Num
 rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: 
NONE","OperatorId:":"SEL_3","children":{"HashTable Sink 
Operator":{"keys:":{"0":"_c
 ol0 (type: int)","1":"_col0 (type: 
int)"},"OperatorId:":"HASHTABLESINK_26"}},"$hdt$_2:hd":{"TableScan":{"alias:":"hd","columns:":["cstring1"],"database:":"default","Statistics:":"Num
 rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: 
NONE","table:":"small_alltypesorc_a","isTempTable:":"false","OperatorId:":"TS_4","children":{"Select
 Operator":{"expressions:":"cstring1 (type: 
string)","columnExprMap:":{"_col0":"cstring1"},"outputColumnNames:":["_col0"],"Statistics:":"Num
 rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: 
NONE","OperatorId:":"SEL_5","children":{"HashTable Sink 
Operator":{"keys:":{"0":"_col1 (type: string)","1":"_col0 (type: 
string)"},"OperatorId:":"HASHTABLESINK_24"},"Stage-3":{"Map 
Reduce":{"Map Operator 
Tree:":[{"TableScan":{"alias:":"c","columns:":["cint","cstring1"],"database:":"default","Statistics:":"Num
 rows: 20 Data size: 4400 Basic stats: COMPLETE Column stats: 
NONE","table:":"small_alltypesorc_a","TableScan Vectorization:":{"n
 ative:":"true","vectorizationSchemaColumns:":"[0:ctinyint:tinyint, 
1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 
5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 
8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 
11:cboolean2:boolean, 
12:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_0","children":{"Select
 Operator":{"expressions:":"cint (type: int), cstring1 (type: 
string)","columnExprMap:":{"_col0":"cint","_col1":"cstring1"},"outputColumnNames:":["_col0","_col1"],"Select
 
Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2,
 6]"},"Statistics:":"Num rows: 20 Data size: 4400 Basic stats: COMPLETE Column 
stats: NONE","OperatorId:":"SEL_28","children":{"Map Join 
Operator":{"columnExprMap:":{"_col1":"0:_col1"},"condition map:":[{"":"Left 
Outer Join 0 to 1"}],"keys:":{"0":"_col0 (type: int)","1":"_col0 (type: 
int)"},"Map Join Vect
 orization:":{"bigTableKeyExpressions:":["col 
2:int"],"bigTableValueExpressions:":["col 
6:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable
 IS true","hive.vectorized.execution.mapjoin.native.enabled IS true","One 
MapJoin Condition IS true","No nullsafe IS true","Small table vectorizes IS 
true","Optimized Table and Supports Key Types IS 
true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS 
false"]},"outputColumnNames:":["_col1"],"Statistics:":"Num rows: 22 Data size: 
4840 Basic stats: COMPLETE Column stats: 
NONE","OperatorId:":"MAPJOIN_29","children":{"Map Join Operator":{"condition 
map:":[{"":"Left Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: 
string)","1":"_col0 (type: string)"},"Map Join 
Vectorization:":{"bigTableKeyExpressions:":["col 
0:string"],"className:":"VectorMapJoinOperator","native:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable
 IS true","hive.vectorized.e
 xecution.mapjoin.native.enabled IS true","One MapJoin Condition IS true","No 
nullsafe IS true","Small table vectorizes IS true","Optimized Table and 
Supports Key Types IS true"],"nativeConditionsNotMet:":["hive.execution.engine 
mr IN [tez, spark] IS false"]},"Statistics:":"Num rows: 24 Data size: 5324 
Basic stats: COMPLETE Column

[1/3] hive git commit: HIVE-18769: Vectorization: Disable vectorization of key-less outer joins (Matt McCline, reviewed by Gopal Vijayaraghavan)

2018-02-22 Thread mmccline 
Repository: hive
Updated Branches:
  refs/heads/master 7a8f10599 -> 1593c10dd


http://git-wip-us.apache.org/repos/asf/hive/blob/1593c10d/ql/src/test/results/clientpositive/vector_outer_join6.q.out
--
diff --git a/ql/src/test/results/clientpositive/vector_outer_join6.q.out 
b/ql/src/test/results/clientpositive/vector_outer_join6.q.out
index 5bd2bac..1e7e295 100644
--- a/ql/src/test/results/clientpositive/vector_outer_join6.q.out
+++ b/ql/src/test/results/clientpositive/vector_outer_join6.q.out
@@ -130,7 +130,7 @@ POSTHOOK: query: explain vectorization detail formatted
 select tj1rnum, tj2rnum, tjoin3.rnum as rnumt3 from
(select tjoin1.rnum tj1rnum, tjoin2.rnum tj2rnum, tjoin2.c1 tj2c1 from 
tjoin1 left outer join tjoin2 on tjoin1.c1 = tjoin2.c1 ) tj left outer join 
tjoin3 on tj2c1 = tjoin3.c1
 POSTHOOK: type: QUERY
-{"PLAN 
VECTORIZATION":{"enabled":true,"enabledConditionsMet":["hive.vectorized.execution.enabled
 IS true"]},"STAGE DEPENDENCIES":{"Stage-7":{"ROOT 
STAGE":"TRUE"},"Stage-5":{"DEPENDENT STAGES":"Stage-7"},"Stage-0":{"DEPENDENT 
STAGES":"Stage-5"}},"STAGE PLANS":{"Stage-7":{"Map Reduce Local Work":{"Alias 
-> Map Local Tables:":{"$hdt$_0:$hdt$_0:tjoin2":{"Fetch 
Operator":{"limit:":"-1"}},"$hdt$_1:tjoin3":{"Fetch 
Operator":{"limit:":"-1"}}},"Alias -> Map Local Operator 
Tree:":{"$hdt$_0:$hdt$_0:tjoin2":{"TableScan":{"alias:":"tjoin2","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num
 rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: 
NONE","table:":"tjoin2","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select
 Operator":{"expressions:":"rnum (type: int), c1 (type: 
int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num
 rows: 4 Data size: 372 Basic stats: COMPLETE Column stats: 
NONE","OperatorId:":"SEL_1
 ","children":{"HashTable Sink Operator":{"keys:":{"0":"_col1 (type: 
int)","1":"_col1 (type: 
int)"},"OperatorId:":"HASHTABLESINK_21"}},"$hdt$_1:tjoin3":{"TableScan":{"alias:":"tjoin3","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num
 rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: 
NONE","table:":"tjoin3","isTempTable:":"false","OperatorId:":"TS_8","children":{"Select
 Operator":{"expressions:":"rnum (type: int), c1 (type: 
int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Statistics:":"Num
 rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: 
NONE","OperatorId:":"SEL_9","children":{"HashTable Sink 
Operator":{"keys:":{"0":"_col2 (type: int)","1":"_col1 (type: 
int)"},"OperatorId:":"HASHTABLESINK_19"},"Stage-5":{"Map Reduce":{"Map 
Operator 
Tree:":[{"TableScan":{"alias:":"tjoin1","columns:":["rnum","c1"],"database:":"default","Statistics:":"Num
 rows: 3 Data size: 32 Basic stats: COMPLETE Column stats: NO
 NE","table:":"tjoin1","TableScan 
Vectorization:":{"native:":"true","vectorizationSchemaColumns:":"[0:rnum:int, 
1:c1:int, 2:c2:int, 
3:ROW__ID:struct]"},"isTempTable:":"false","OperatorId:":"TS_2","children":{"Select
 Operator":{"expressions:":"rnum (type: int), c1 (type: 
int)","columnExprMap:":{"_col0":"rnum","_col1":"c1"},"outputColumnNames:":["_col0","_col1"],"Select
 
Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[0,
 1]"},"Statistics:":"Num rows: 3 Data size: 32 Basic stats: COMPLETE Column 
stats: NONE","OperatorId:":"SEL_23","children":{"Map Join 
Operator":{"columnExprMap:":{"_col0":"0:_col0","_col1":"0:_col1","_col2":"1:_col0"},"condition
 map:":[{"":"Right Outer Join 0 to 1"}],"keys:":{"0":"_col1 (type: 
int)","1":"_col1 (type: int)"},"Map Join 
Vectorization:":{"bigTableKeyExpressions:":["col 
1:int"],"bigTableValueExpressions:":["col 
0:int"],"className:":"VectorMapJoinOperator","nat
 ive:":"false","nativeConditionsMet:":["hive.mapjoin.optimized.hashtable IS 
true","hive.vectorized.execution.mapjoin.native.enabled IS true","One MapJoin 
Condition IS true","No nullsafe IS true","Small table vectorizes IS 
true","Optimized Table and Supports Key Types IS 
true"],"nativeConditionsNotMet:":["hive.execution.engine mr IN [tez, spark] IS 
false"]},"outputColumnNames:":["_col0","_col1","_col2"],"Statistics:":"Num 
rows: 4 Data size: 409 Basic stats: COMPLETE Column stats: 
NONE","OperatorId:":"MAPJOIN_24","children":{"Select 
Operator":{"expressions:":"_col2 (type: int), _col0 (type: int), _col1 (type: 
int)","columnExprMap:":{"_col0":"_col2","_col1":"_col0","_col2":"_col1"},"outputColumnNames:":["_col0","_col1","_col2"],"Select
 
Vectorization:":{"className:":"VectorSelectOperator","native:":"true","projectedOutputColumnNums:":"[2,
 0, 1]"},"Statistics:":"Num rows: 4 Data size: 409 Basic stats: COMPLETE Column 
stats: NONE","OperatorId:":"SEL_25","children":{"Map Join Operator":{"co

[3/3] hive git commit: HIVE-18769: Vectorization: Disable vectorization of key-less outer joins (Matt McCline, reviewed by Gopal Vijayaraghavan)

2018-02-22 Thread mmccline 
HIVE-18769: Vectorization: Disable vectorization of key-less outer joins (Matt 
McCline, reviewed by Gopal Vijayaraghavan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1593c10d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1593c10d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1593c10d

Branch: refs/heads/master
Commit: 1593c10dd4b917bf0736779e458739b306835a3f
Parents: 7a8f105
Author: Matt McCline 
Authored: Thu Feb 22 19:07:21 2018 -0800
Committer: Matt McCline 
Committed: Thu Feb 22 19:07:21 2018 -0800

--
 .../hive/ql/optimizer/physical/Vectorizer.java  |   6 +-
 .../apache/hadoop/hive/ql/plan/MapJoinDesc.java |   6 +
 .../hadoop/hive/ql/plan/VectorMapJoinDesc.java  |   7 +
 .../clientpositive/vector_outer_join_no_keys.q  |  21 +
 .../clientpositive/llap/vector_coalesce_3.q.out |   2 +-
 .../clientpositive/llap/vector_join30.q.out |   4 +-
 .../llap/vector_left_outer_join2.q.out  |   8 +-
 .../llap/vector_leftsemi_mapjoin.q.out  |  16 +-
 .../llap/vector_outer_join0.q.out   |   4 +-
 .../llap/vector_outer_join1.q.out   |   8 +-
 .../llap/vector_outer_join2.q.out   |   4 +-
 .../llap/vector_outer_join_no_keys.q.out| 408 +++
 ...k_vectorized_dynamic_partition_pruning.q.out |   4 +-
 .../spark/vector_outer_join0.q.out  |   4 +-
 .../spark/vector_outer_join1.q.out  |   8 +-
 .../spark/vector_outer_join2.q.out  |   4 +-
 .../clientpositive/vector_coalesce_3.q.out  |   2 +-
 .../clientpositive/vector_groupby_mapjoin.q.out |   2 +-
 .../vector_left_outer_join2.q.out   |   8 +-
 .../clientpositive/vector_outer_join0.q.out |   4 +-
 .../clientpositive/vector_outer_join1.q.out |   8 +-
 .../clientpositive/vector_outer_join2.q.out |   4 +-
 .../clientpositive/vector_outer_join3.q.out |   6 +-
 .../clientpositive/vector_outer_join4.q.out |   6 +-
 .../clientpositive/vector_outer_join6.q.out |   4 +-
 .../vector_outer_join_no_keys.q.out | 317 ++
 26 files changed, 819 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1593c10d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 52ef2d3..783a672 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -3150,6 +3150,8 @@ public class Vectorizer implements PhysicalPlanResolver {
 
 List keyDesc = desc.getKeys().get(posBigTable);
 
+boolean outerJoinHasNoKeys = (!desc.isNoOuterJoin() && keyDesc.size() == 
0);
+
 // For now, we don't support joins on or using DECIMAL_64.
 VectorExpression[] allBigTableKeyExpressions =
 vContext.getVectorExpressionsUpConvertDecimal64(keyDesc);
@@ -3452,6 +3454,7 @@ public class Vectorizer implements PhysicalPlanResolver {
 vectorDesc.setOneMapJoinCondition(oneMapJoinCondition);
 vectorDesc.setHasNullSafes(hasNullSafes);
 vectorDesc.setSmallTableExprVectorizes(smallTableExprVectorizes);
+vectorDesc.setOuterJoinHasNoKeys(outerJoinHasNoKeys);
 
 vectorDesc.setIsFastHashTableEnabled(isFastHashTableEnabled);
 vectorDesc.setIsHybridHashJoin(isHybridHashJoin);
@@ -3468,7 +3471,8 @@ public class Vectorizer implements PhysicalPlanResolver {
 !isTezOrSpark ||
 !oneMapJoinCondition ||
 hasNullSafes ||
-!smallTableExprVectorizes) {
+!smallTableExprVectorizes ||
+outerJoinHasNoKeys) {
   result = false;
 }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/1593c10d/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 
b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
index 23dd19b..cf4ab60 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
@@ -444,6 +444,12 @@ public class MapJoinDesc extends JoinDesc implements 
Serializable {
   new VectorizationCondition(
   vectorMapJoinDesc.getSmallTableExprVectorizes(),
   "Small table vectorizes"));
+  if (!mapJoinDesc.isNoOuterJoin()) {
+conditionList.add(
+new VectorizationCondition(
+!vectorMapJoinDesc.getOuterJoinHasNoKeys(),
+

hive git commit: HIVE-18449 : Add configurable policy for choosing the HMS URI from hive.metastore.uris (Janaki Lahorani, reviewed by Vihang Karajgaonkar)

2018-02-22 Thread vihangk1 
Repository: hive
Updated Branches:
  refs/heads/branch-2 042296fbc -> d72c35e47


HIVE-18449 : Add configurable policy for choosing the HMS URI from 
hive.metastore.uris (Janaki Lahorani, reviewed by Vihang Karajgaonkar)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d72c35e4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d72c35e4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d72c35e4

Branch: refs/heads/branch-2
Commit: d72c35e478c68898877b93f7294f178bfcdd0c87
Parents: 042296f
Author: Vihang Karajgaonkar 
Authored: Thu Feb 22 17:44:09 2018 -0800
Committer: Vihang Karajgaonkar 
Committed: Thu Feb 22 17:44:09 2018 -0800

--
 .../org/apache/hadoop/hive/conf/HiveConf.java |  7 +++
 .../hive/metastore/HiveMetaStoreClient.java   | 18 +++---
 2 files changed, 18 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/d72c35e4/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4a92ffc..df2aee8 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -212,6 +212,7 @@ public class HiveConf extends Configuration {
   HiveConf.ConfVars.METASTOREWAREHOUSE,
   HiveConf.ConfVars.REPLDIR,
   HiveConf.ConfVars.METASTOREURIS,
+  HiveConf.ConfVars.METASTORESELECTION,
   HiveConf.ConfVars.METASTORE_SERVER_PORT,
   HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES,
   HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES,
@@ -598,6 +599,12 @@ public class HiveConf extends Configuration {
 
 METASTORE_CAPABILITY_CHECK("hive.metastore.client.capability.check", true,
 "Whether to check client capabilities for potentially breaking API 
usage."),
+METASTORESELECTION("hive.metastore.uri.selection", "RANDOM",
+new StringSet("SEQUENTIAL", "RANDOM"),
+"Determines the selection mechanism used by metastore client to 
connect to remote " +
+"metastore.  SEQUENTIAL implies that the first valid metastore from 
the URIs specified " +
+"as part of hive.metastore.uris will be picked.  RANDOM implies that 
the metastore " +
+"will be picked randomly"),
 METASTORE_FASTPATH("hive.metastore.fastpath", false,
 "Used to avoid all of the proxies and object copies in the metastore.  
Note, if this is " +
 "set, you MUST use a local metastore (hive.metastore.uris must be 
empty) otherwise " +

http://git-wip-us.apache.org/repos/asf/hive/blob/d72c35e4/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
--
diff --git 
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java 
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 7dcd13e..83c2860 100644
--- 
a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ 
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -203,9 +203,11 @@ public class HiveMetaStoreClient implements 
IMetaStoreClient {
 
 }
 // make metastore URIS random
-List uriList = Arrays.asList(metastoreUris);
-Collections.shuffle(uriList);
-metastoreUris = (URI[]) uriList.toArray();
+if (HiveConf.getVar(conf, 
ConfVars.METASTORESELECTION).equalsIgnoreCase("RANDOM")) {
+  List uriList = Arrays.asList(metastoreUris);
+  Collections.shuffle(uriList);
+  metastoreUris = (URI[]) uriList.toArray();
+}
   } catch (IllegalArgumentException e) {
 throw (e);
   } catch (Exception e) {
@@ -341,10 +343,12 @@ public class HiveMetaStoreClient implements 
IMetaStoreClient {
   " at the client level.");
 } else {
   close();
-  // Swap the first element of the metastoreUris[] with a random element 
from the rest
-  // of the array. Rationale being that this method will generally be 
called when the default
-  // connection has died and the default connection is likely to be the 
first array element.
-  promoteRandomMetaStoreURI();
+  if (HiveConf.getVar(conf, 
ConfVars.METASTORESELECTION).equalsIgnoreCase("RANDOM")) {
+// Swap the first element of the metastoreUris[] with a random element 
from the rest
+// of the array. Rationale being that this method will generally be 
called when the default
+// connection has died and the default connection is likely to be the 
first array element.
+

hive git commit: HIVE-18713: addendum; fix q.out mid-air conflict

2018-02-22 Thread gopalv 
Repository: hive
Updated Branches:
  refs/heads/master 2006fb8d2 -> 7a8f10599


HIVE-18713: addendum; fix q.out mid-air conflict


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7a8f1059
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7a8f1059
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7a8f1059

Branch: refs/heads/master
Commit: 7a8f105999fb6bc2dc9e444588d78f848002dca4
Parents: 2006fb8
Author: Gopal V 
Authored: Thu Feb 22 13:39:56 2018 -0800
Committer: Gopal V 
Committed: Thu Feb 22 13:40:09 2018 -0800

--
 .../llap/vectorized_mapjoin3.q.out  | 100 ---
 1 file changed, 65 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/7a8f1059/ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out 
b/ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out
index 4fb3466..00286ce 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_mapjoin3.q.out
@@ -43,7 +43,6 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 POSTHOOK: Output: default@table_6
 POSTHOOK: Lineage: table_6.int_col_0 SCRIPT []
-Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product
 PREHOOK: query: SELECT t1.decimal0801_col
 FROM table_19 t1
 WHERE (SELECT max(tt1.int_col_0) AS int_col FROM table_6 tt1) IN 
(t1.int_col_1) AND decimal0801_col is not null
@@ -89,7 +88,6 @@ POSTHOOK: Input: default@table_19
 POSTHOOK: Input: default@table_6
  A masked pattern was here 
 418.9
-Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product
 PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
 SELECT t1.decimal0801_col
 FROM table_19 t1
@@ -122,45 +120,69 @@ STAGE PLANS:
 TableScan
   alias: t1
   Statistics: Num rows: 5 Data size: 580 Basic stats: COMPLETE 
Column stats: COMPLETE
+  TableScan Vectorization:
+  native: true
+  vectorizationSchemaColumns: 
[0:decimal0801_col:decimal(8,1), 1:int_col_1:int, 
2:ROW__ID:struct]
   Filter Operator
-predicate: decimal0801_col is not null (type: boolean)
-Statistics: Num rows: 5 Data size: 580 Basic stats: 
COMPLETE Column stats: COMPLETE
+Filter Vectorization:
+className: VectorFilterOperator
+native: true
+predicateExpression: FilterExprAndExpr(children: 
SelectColumnIsNotNull(col 0:decimal(8,1)), SelectColumnIsNotNull(col 1:int))
+predicate: (decimal0801_col is not null and int_col_1 is 
not null) (type: boolean)
+Statistics: Num rows: 4 Data size: 464 Basic stats: 
COMPLETE Column stats: COMPLETE
 Select Operator
   expressions: decimal0801_col (type: decimal(8,1)), 
int_col_1 (type: int)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 5 Data size: 580 Basic stats: 
COMPLETE Column stats: COMPLETE
+  Select Vectorization:
+  className: VectorSelectOperator
+  native: true
+  projectedOutputColumnNums: [0, 1]
+  Statistics: Num rows: 4 Data size: 464 Basic stats: 
COMPLETE Column stats: COMPLETE
   Map Join Operator
 condition map:
- Left Outer Join 0 to 1
+ Inner Join 0 to 1
 keys:
-  0 
-  1 
-outputColumnNames: _col0, _col1, _col2
+  0 _col1 (type: int)
+  1 _col0 (type: int)
+Map Join Vectorization:
+bigTableKeyColumnNums: [1]
+bigTableRetainedColumnNums: [0]
+bigTableValueColumnNums: [0]
+className: VectorMapJoinInnerBigOnlyLongOperator
+native: true
+nativeConditionsMet: 
hive.mapjoin.optimized.hashtable IS true, 
hive.vectorized.execution.mapjoin.native.enabled IS true, hive.execution.engine 
tez IN [tez, spark] IS true, One MapJoin Condition IS true, No nullsafe IS 
true, Small table vectorizes IS true, Optimized Table

hive git commit: HIVE-18756: addendum; stat changes are unconnected

2018-02-22 Thread kgyrtkirk 
Repository: hive
Updated Branches:
  refs/heads/master ec2378f53 -> 2006fb8d2


HIVE-18756: addendum; stat changes are unconnected


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2006fb8d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2006fb8d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2006fb8d

Branch: refs/heads/master
Commit: 2006fb8d26c1715fcb89fdd1323e245a802418be
Parents: ec2378f
Author: Zoltan Haindrich 
Authored: Thu Feb 22 17:59:34 2018 +0100
Committer: Zoltan Haindrich 
Committed: Thu Feb 22 17:59:34 2018 +0100

--
 .../llap/vectorization_part_project.q.out   | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/2006fb8d/ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out 
b/ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out
index 130e137..e46c7f4 100644
--- a/ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorization_part_project.q.out
@@ -70,15 +70,15 @@ STAGE PLANS:
 Map Operator Tree:
 TableScan
   alias: alltypesorc_part
-  Statistics: Num rows: 200 Data size: 1600 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 200 Data size: 1592 Basic stats: 
COMPLETE Column stats: COMPLETE
   Select Operator
 expressions: (cdouble + 2.0) (type: double)
 outputColumnNames: _col0
-Statistics: Num rows: 200 Data size: 1600 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 200 Data size: 1600 Basic stats: 
COMPLETE Column stats: COMPLETE
 Reduce Output Operator
   key expressions: _col0 (type: double)
   sort order: +
-  Statistics: Num rows: 200 Data size: 1600 Basic stats: 
COMPLETE Column stats: PARTIAL
+  Statistics: Num rows: 200 Data size: 1600 Basic stats: 
COMPLETE Column stats: COMPLETE
   TopN Hash Memory Usage: 0.1
 Execution mode: vectorized, llap
 LLAP IO: all inputs
@@ -103,13 +103,13 @@ STAGE PLANS:
   Select Operator
 expressions: KEY.reducesinkkey0 (type: double)
 outputColumnNames: _col0
-Statistics: Num rows: 200 Data size: 1600 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 200 Data size: 1600 Basic stats: 
COMPLETE Column stats: COMPLETE
 Limit
   Number of rows: 10
-  Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE 
Column stats: PARTIAL
+  Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE 
Column stats: COMPLETE
   File Output Operator
 compressed: false
-Statistics: Num rows: 10 Data size: 80 Basic stats: 
COMPLETE Column stats: PARTIAL
+Statistics: Num rows: 10 Data size: 80 Basic stats: 
COMPLETE Column stats: COMPLETE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat