[11/48] hive git commit: HIVE-13826: Make VectorUDFAdaptor work for GenericUDFBetween when used as FILTER (Matt McCline, reviewed by Ashutosh Chauhan)

2016-05-27 Thread spena 
HIVE-13826: Make VectorUDFAdaptor work for GenericUDFBetween when used as 
FILTER (Matt McCline, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/71725869
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/71725869
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/71725869

Branch: refs/heads/java8
Commit: 7172586966739de0ba1659bf9abcea40d109b341
Parents: 9bebaf6
Author: Matt McCline 
Authored: Thu May 26 08:25:29 2016 -0700
Committer: Matt McCline 
Committed: Thu May 26 08:25:29 2016 -0700

--
 .../ql/exec/vector/VectorizationContext.java|  22 +++-
 .../clientpositive/vector_between_columns.q |   8 +-
 .../tez/vector_between_columns.q.out| 116 +-
 .../clientpositive/vector_between_columns.q.out | 117 ++-
 4 files changed, 250 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/71725869/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 886e222..a76e31d 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -1965,8 +1965,17 @@ public class VectorizationContext {
   private VectorExpression getCustomUDFExpression(ExprNodeGenericFuncDesc 
expr,  Mode mode)
   throws HiveException {
 
-if (mode != Mode.PROJECTION) {
-  return null;
+boolean isFilter = false;// Assume.
+if (mode == Mode.FILTER) {
+
+  // Is output type a BOOLEAN?
+  TypeInfo resultTypeInfo = expr.getTypeInfo();
+  if (resultTypeInfo.getCategory() == Category.PRIMITIVE &&
+  ((PrimitiveTypeInfo) resultTypeInfo).getPrimitiveCategory() == 
PrimitiveCategory.BOOLEAN) {
+isFilter = true;
+  } else {
+return null;
+  }
 }
 
 //GenericUDFBridge udfBridge = (GenericUDFBridge) expr.getGenericUDF();
@@ -2032,7 +2041,14 @@ public class VectorizationContext {
 for (Integer i : exprResultColumnNums) {
   ocm.freeOutputColumn(i);
 }
-return ve;
+
+if (isFilter) {
+  SelectColumnIsTrue filterVectorExpr = new SelectColumnIsTrue(outputCol);
+  filterVectorExpr.setChildExpressions(new VectorExpression[] {ve});
+  return filterVectorExpr;
+} else {
+  return ve;
+}
   }
 
   public static boolean isStringFamily(String resultType) {

http://git-wip-us.apache.org/repos/asf/hive/blob/71725869/ql/src/test/queries/clientpositive/vector_between_columns.q
--
diff --git a/ql/src/test/queries/clientpositive/vector_between_columns.q 
b/ql/src/test/queries/clientpositive/vector_between_columns.q
index ba38445..41f9243 100644
--- a/ql/src/test/queries/clientpositive/vector_between_columns.q
+++ b/ql/src/test/queries/clientpositive/vector_between_columns.q
@@ -7,8 +7,7 @@ set hive.mapred.mode=nonstrict;
 
 -- SORT_QUERY_RESULTS
 --
--- The following WILL NOT BE ABLE TO USE the VectorUDFAdaptor to 
GenericUDFBetween
--- because the mode = FILTER is not supported yet.
+-- Verify the VectorUDFAdaptor to GenericUDFBetween works for PROJECTION and 
FILTER.
 --
 create table if not exists TSINT_txt ( RNUM int , CSINT smallint )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n';
@@ -25,6 +24,11 @@ create table TSINT stored as orc AS SELECT * FROM TSINT_txt;
 create table TINT stored as orc AS SELECT * FROM TINT_txt;
 
 
+explain
+select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint 
between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col 
from tint , tsint;
+
+select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint 
between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col 
from tint , tsint;
+
 
 explain
 select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where 
tint.cint between tsint.csint and tsint.csint;

http://git-wip-us.apache.org/repos/asf/hive/blob/71725869/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out 
b/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out
index 8a9978b..939aab5 100644
--- a/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out
+++

hive git commit: HIVE-13826: Make VectorUDFAdaptor work for GenericUDFBetween when used as FILTER (Matt McCline, reviewed by Ashutosh Chauhan)

2016-05-26 Thread mmccline 
Repository: hive
Updated Branches:
  refs/heads/branch-2.1 ffbd633fa -> f547a6420


HIVE-13826: Make VectorUDFAdaptor work for GenericUDFBetween when used as 
FILTER (Matt McCline, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f547a642
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f547a642
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f547a642

Branch: refs/heads/branch-2.1
Commit: f547a6420914caebb2e8c842f0244d785b3910a9
Parents: ffbd633
Author: Matt McCline 
Authored: Thu May 26 08:25:29 2016 -0700
Committer: Matt McCline 
Committed: Thu May 26 10:10:57 2016 -0700

--
 .../ql/exec/vector/VectorizationContext.java|  22 +++-
 .../clientpositive/vector_between_columns.q |   8 +-
 .../tez/vector_between_columns.q.out| 116 +-
 .../clientpositive/vector_between_columns.q.out | 117 ++-
 4 files changed, 250 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f547a642/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 886e222..a76e31d 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -1965,8 +1965,17 @@ public class VectorizationContext {
   private VectorExpression getCustomUDFExpression(ExprNodeGenericFuncDesc 
expr,  Mode mode)
   throws HiveException {
 
-if (mode != Mode.PROJECTION) {
-  return null;
+boolean isFilter = false;// Assume.
+if (mode == Mode.FILTER) {
+
+  // Is output type a BOOLEAN?
+  TypeInfo resultTypeInfo = expr.getTypeInfo();
+  if (resultTypeInfo.getCategory() == Category.PRIMITIVE &&
+  ((PrimitiveTypeInfo) resultTypeInfo).getPrimitiveCategory() == 
PrimitiveCategory.BOOLEAN) {
+isFilter = true;
+  } else {
+return null;
+  }
 }
 
 //GenericUDFBridge udfBridge = (GenericUDFBridge) expr.getGenericUDF();
@@ -2032,7 +2041,14 @@ public class VectorizationContext {
 for (Integer i : exprResultColumnNums) {
   ocm.freeOutputColumn(i);
 }
-return ve;
+
+if (isFilter) {
+  SelectColumnIsTrue filterVectorExpr = new SelectColumnIsTrue(outputCol);
+  filterVectorExpr.setChildExpressions(new VectorExpression[] {ve});
+  return filterVectorExpr;
+} else {
+  return ve;
+}
   }
 
   public static boolean isStringFamily(String resultType) {

http://git-wip-us.apache.org/repos/asf/hive/blob/f547a642/ql/src/test/queries/clientpositive/vector_between_columns.q
--
diff --git a/ql/src/test/queries/clientpositive/vector_between_columns.q 
b/ql/src/test/queries/clientpositive/vector_between_columns.q
index ba38445..41f9243 100644
--- a/ql/src/test/queries/clientpositive/vector_between_columns.q
+++ b/ql/src/test/queries/clientpositive/vector_between_columns.q
@@ -7,8 +7,7 @@ set hive.mapred.mode=nonstrict;
 
 -- SORT_QUERY_RESULTS
 --
--- The following WILL NOT BE ABLE TO USE the VectorUDFAdaptor to 
GenericUDFBetween
--- because the mode = FILTER is not supported yet.
+-- Verify the VectorUDFAdaptor to GenericUDFBetween works for PROJECTION and 
FILTER.
 --
 create table if not exists TSINT_txt ( RNUM int , CSINT smallint )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n';
@@ -25,6 +24,11 @@ create table TSINT stored as orc AS SELECT * FROM TSINT_txt;
 create table TINT stored as orc AS SELECT * FROM TINT_txt;
 
 
+explain
+select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint 
between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col 
from tint , tsint;
+
+select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint 
between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col 
from tint , tsint;
+
 
 explain
 select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where 
tint.cint between tsint.csint and tsint.csint;

http://git-wip-us.apache.org/repos/asf/hive/blob/f547a642/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out 
b/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out
index 8a9978b..939aab5 100644
---

hive git commit: HIVE-13826: Make VectorUDFAdaptor work for GenericUDFBetween when used as FILTER (Matt McCline, reviewed by Ashutosh Chauhan)

2016-05-26 Thread mmccline 
Repository: hive
Updated Branches:
  refs/heads/branch-1 0e83ae1e2 -> abaf88248


HIVE-13826: Make VectorUDFAdaptor work for GenericUDFBetween when used as 
FILTER (Matt McCline, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/abaf8824
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/abaf8824
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/abaf8824

Branch: refs/heads/branch-1
Commit: abaf88248d3cc92c157e537ed4510136e003a132
Parents: 0e83ae1
Author: Matt McCline 
Authored: Thu May 26 08:48:08 2016 -0700
Committer: Matt McCline 
Committed: Thu May 26 08:48:08 2016 -0700

--
 .../ql/exec/vector/VectorizationContext.java|  22 +++-
 .../clientpositive/vector_between_columns.q |   8 +-
 .../tez/vector_between_columns.q.out| 115 +-
 .../clientpositive/vector_between_columns.q.out | 117 ++-
 4 files changed, 249 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/abaf8824/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index f09bfd6..86fdf1c 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -1946,8 +1946,17 @@ public class VectorizationContext {
   private VectorExpression getCustomUDFExpression(ExprNodeGenericFuncDesc 
expr,  Mode mode)
   throws HiveException {
 
-if (mode != Mode.PROJECTION) {
-  return null;
+boolean isFilter = false;// Assume.
+if (mode == Mode.FILTER) {
+
+  // Is output type a BOOLEAN?
+  TypeInfo resultTypeInfo = expr.getTypeInfo();
+  if (resultTypeInfo.getCategory() == Category.PRIMITIVE &&
+  ((PrimitiveTypeInfo) resultTypeInfo).getPrimitiveCategory() == 
PrimitiveCategory.BOOLEAN) {
+isFilter = true;
+  } else {
+return null;
+  }
 }
 
 //GenericUDFBridge udfBridge = (GenericUDFBridge) expr.getGenericUDF();
@@ -2018,7 +2027,14 @@ public class VectorizationContext {
 for (Integer i : exprResultColumnNums) {
   ocm.freeOutputColumn(i);
 }
-return ve;
+
+if (isFilter) {
+  SelectColumnIsTrue filterVectorExpr = new SelectColumnIsTrue(outputCol);
+  filterVectorExpr.setChildExpressions(new VectorExpression[] {ve});
+  return filterVectorExpr;
+} else {
+  return ve;
+}
   }
 
   public static boolean isStringFamily(String resultType) {

http://git-wip-us.apache.org/repos/asf/hive/blob/abaf8824/ql/src/test/queries/clientpositive/vector_between_columns.q
--
diff --git a/ql/src/test/queries/clientpositive/vector_between_columns.q 
b/ql/src/test/queries/clientpositive/vector_between_columns.q
index ba38445..41f9243 100644
--- a/ql/src/test/queries/clientpositive/vector_between_columns.q
+++ b/ql/src/test/queries/clientpositive/vector_between_columns.q
@@ -7,8 +7,7 @@ set hive.mapred.mode=nonstrict;
 
 -- SORT_QUERY_RESULTS
 --
--- The following WILL NOT BE ABLE TO USE the VectorUDFAdaptor to 
GenericUDFBetween
--- because the mode = FILTER is not supported yet.
+-- Verify the VectorUDFAdaptor to GenericUDFBetween works for PROJECTION and 
FILTER.
 --
 create table if not exists TSINT_txt ( RNUM int , CSINT smallint )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n';
@@ -25,6 +24,11 @@ create table TSINT stored as orc AS SELECT * FROM TSINT_txt;
 create table TINT stored as orc AS SELECT * FROM TINT_txt;
 
 
+explain
+select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint 
between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col 
from tint , tsint;
+
+select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint 
between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col 
from tint , tsint;
+
 
 explain
 select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where 
tint.cint between tsint.csint and tsint.csint;

http://git-wip-us.apache.org/repos/asf/hive/blob/abaf8824/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out 
b/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out
index 1361d2d..e7af0db 100644
---

hive git commit: HIVE-13826: Make VectorUDFAdaptor work for GenericUDFBetween when used as FILTER (Matt McCline, reviewed by Ashutosh Chauhan)

2016-05-26 Thread mmccline 
Repository: hive
Updated Branches:
  refs/heads/master 9bebaf619 -> 717258696


HIVE-13826: Make VectorUDFAdaptor work for GenericUDFBetween when used as 
FILTER (Matt McCline, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/71725869
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/71725869
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/71725869

Branch: refs/heads/master
Commit: 7172586966739de0ba1659bf9abcea40d109b341
Parents: 9bebaf6
Author: Matt McCline 
Authored: Thu May 26 08:25:29 2016 -0700
Committer: Matt McCline 
Committed: Thu May 26 08:25:29 2016 -0700

--
 .../ql/exec/vector/VectorizationContext.java|  22 +++-
 .../clientpositive/vector_between_columns.q |   8 +-
 .../tez/vector_between_columns.q.out| 116 +-
 .../clientpositive/vector_between_columns.q.out | 117 ++-
 4 files changed, 250 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/71725869/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 886e222..a76e31d 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -1965,8 +1965,17 @@ public class VectorizationContext {
   private VectorExpression getCustomUDFExpression(ExprNodeGenericFuncDesc 
expr,  Mode mode)
   throws HiveException {
 
-if (mode != Mode.PROJECTION) {
-  return null;
+boolean isFilter = false;// Assume.
+if (mode == Mode.FILTER) {
+
+  // Is output type a BOOLEAN?
+  TypeInfo resultTypeInfo = expr.getTypeInfo();
+  if (resultTypeInfo.getCategory() == Category.PRIMITIVE &&
+  ((PrimitiveTypeInfo) resultTypeInfo).getPrimitiveCategory() == 
PrimitiveCategory.BOOLEAN) {
+isFilter = true;
+  } else {
+return null;
+  }
 }
 
 //GenericUDFBridge udfBridge = (GenericUDFBridge) expr.getGenericUDF();
@@ -2032,7 +2041,14 @@ public class VectorizationContext {
 for (Integer i : exprResultColumnNums) {
   ocm.freeOutputColumn(i);
 }
-return ve;
+
+if (isFilter) {
+  SelectColumnIsTrue filterVectorExpr = new SelectColumnIsTrue(outputCol);
+  filterVectorExpr.setChildExpressions(new VectorExpression[] {ve});
+  return filterVectorExpr;
+} else {
+  return ve;
+}
   }
 
   public static boolean isStringFamily(String resultType) {

http://git-wip-us.apache.org/repos/asf/hive/blob/71725869/ql/src/test/queries/clientpositive/vector_between_columns.q
--
diff --git a/ql/src/test/queries/clientpositive/vector_between_columns.q 
b/ql/src/test/queries/clientpositive/vector_between_columns.q
index ba38445..41f9243 100644
--- a/ql/src/test/queries/clientpositive/vector_between_columns.q
+++ b/ql/src/test/queries/clientpositive/vector_between_columns.q
@@ -7,8 +7,7 @@ set hive.mapred.mode=nonstrict;
 
 -- SORT_QUERY_RESULTS
 --
--- The following WILL NOT BE ABLE TO USE the VectorUDFAdaptor to 
GenericUDFBetween
--- because the mode = FILTER is not supported yet.
+-- Verify the VectorUDFAdaptor to GenericUDFBetween works for PROJECTION and 
FILTER.
 --
 create table if not exists TSINT_txt ( RNUM int , CSINT smallint )
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n';
@@ -25,6 +24,11 @@ create table TSINT stored as orc AS SELECT * FROM TSINT_txt;
 create table TINT stored as orc AS SELECT * FROM TINT_txt;
 
 
+explain
+select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint 
between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col 
from tint , tsint;
+
+select tint.rnum, tsint.rnum, tint.cint, tsint.csint, (case when (tint.cint 
between tsint.csint and tsint.csint) then "Ok" else "NoOk" end) as between_col 
from tint , tsint;
+
 
 explain
 select tint.rnum, tsint.rnum, tint.cint, tsint.csint from tint , tsint where 
tint.cint between tsint.csint and tsint.csint;

http://git-wip-us.apache.org/repos/asf/hive/blob/71725869/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out 
b/ql/src/test/results/clientpositive/tez/vector_between_columns.q.out
index 8a9978b..939aab5 100644
---