hive git commit: HIVE-13867 restore HiveAuthorizer interface changes (Thejas Nair, reviewed by Sushanth Sowmyan)

[khorgath]

Repository: hive
Updated Branches:
  refs/heads/master 01c9f0875 -> 181a85b85


HIVE-13867 restore HiveAuthorizer interface changes (Thejas Nair, reviewed by 
Sushanth Sowmyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/181a85b8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/181a85b8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/181a85b8

Branch: refs/heads/master
Commit: 181a85b85c8b7819b4f75845eeea4b053e26e3ee
Parents: 01c9f08
Author: Sushanth Sowmyan 
Authored: Tue May 31 19:31:52 2016 -0700
Committer: Sushanth Sowmyan 
Committed: Tue May 31 19:44:21 2016 -0700

--
 .../TestHiveAuthorizerCheckInvocation.java  |  2 +-
 .../plugin/TestHiveAuthorizerShowFilters.java   |  4 +-
 .../jdbc/authorization/TestHS2AuthzContext.java | 12 +--
 .../authorization/TestJdbcMetadataApiAuth.java  |  4 +-
 .../hive/ql/security/DummyAuthenticator.java|  5 -
 .../security/InjectableDummyAuthenticator.java  |  5 -
 ...SQLStdHiveAuthorizationValidatorForTest.java |  6 +-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |  5 +-
 .../apache/hadoop/hive/ql/parse/TableMask.java  |  9 +-
 .../hadoop/hive/ql/processors/CommandUtil.java  |  5 +-
 .../ql/security/HadoopDefaultAuthenticator.java |  5 -
 .../ql/security/HiveAuthenticationProvider.java |  2 -
 .../SessionStateConfigUserAuthenticator.java|  5 -
 .../security/SessionStateUserAuthenticator.java |  5 -
 .../AuthorizationMetaStoreFilterHook.java   |  3 +-
 .../plugin/HiveAuthorizationValidator.java  |  6 +-
 .../authorization/plugin/HiveAuthorizer.java|  6 +-
 .../plugin/HiveAuthorizerImpl.java  |  6 +-
 .../authorization/plugin/HiveAuthzContext.java  | 99 
 .../authorization/plugin/HiveV1Authorizer.java  |  6 +-
 .../authorization/plugin/QueryContext.java  | 78 ---
 .../sqlstd/DummyHiveAuthorizationValidator.java |  8 +-
 .../SQLStdHiveAuthorizationValidator.java   |  8 +-
 .../cli/operation/MetadataOperation.java|  5 +-
 24 files changed, 150 insertions(+), 149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/181a85b8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
index 5e601c9..9aca713 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
@@ -425,7 +425,7 @@ public class TestHiveAuthorizerCheckInvocation {
 
 verify(mockedAuthorizer).checkPrivileges(any(HiveOperationType.class),
 inputsCapturer.capture(), outputsCapturer.capture(),
-any(QueryContext.class));
+any(HiveAuthzContext.class));
 
 return new ImmutablePair(inputsCapturer.getValue(), 
outputsCapturer.getValue());
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/181a85b8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
index 0209044..5922a8c 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
@@ -77,7 +77,7 @@ public class TestHiveAuthorizerShowFilters {
 protected abstract class AuthorizerWithFilterCmdImpl implements 
HiveAuthorizer {
   @Override
   public List 
filterListCmdObjects(List listObjs,
-  QueryContext context) throws HiveAuthzPluginException, 
HiveAccessControlException {
+  HiveAuthzContext context) throws HiveAuthzPluginException, 
HiveAccessControlException {
 // capture arguments in static
 filterArguments = listObjs;
 // return static variable with results, if it is set to some set of
@@ -101,7 +101,7 @@ public class TestHiveAuthorizerShowFilters {
   try {
 Mockito.when(
 

hive git commit: HIVE-13867 restore HiveAuthorizer interface changes (Thejas Nair, reviewed by Sushanth Sowmyan)

[khorgath]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 82d284a45 -> 4af1be71e


HIVE-13867 restore HiveAuthorizer interface changes (Thejas Nair, reviewed by 
Sushanth Sowmyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4af1be71
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4af1be71
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4af1be71

Branch: refs/heads/branch-2.1
Commit: 4af1be71ee1068262168c52ea8be6b2a619a5b9e
Parents: 82d284a
Author: Sushanth Sowmyan 
Authored: Tue May 31 19:31:52 2016 -0700
Committer: Sushanth Sowmyan 
Committed: Tue May 31 19:43:52 2016 -0700

--
 .../TestHiveAuthorizerCheckInvocation.java  |  2 +-
 .../plugin/TestHiveAuthorizerShowFilters.java   |  4 +-
 .../jdbc/authorization/TestHS2AuthzContext.java | 12 +--
 .../authorization/TestJdbcMetadataApiAuth.java  |  4 +-
 .../hive/ql/security/DummyAuthenticator.java|  5 -
 .../security/InjectableDummyAuthenticator.java  |  5 -
 ...SQLStdHiveAuthorizationValidatorForTest.java |  6 +-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |  5 +-
 .../apache/hadoop/hive/ql/parse/TableMask.java  |  9 +-
 .../hadoop/hive/ql/processors/CommandUtil.java  |  5 +-
 .../ql/security/HadoopDefaultAuthenticator.java |  5 -
 .../ql/security/HiveAuthenticationProvider.java |  2 -
 .../SessionStateConfigUserAuthenticator.java|  5 -
 .../security/SessionStateUserAuthenticator.java |  5 -
 .../AuthorizationMetaStoreFilterHook.java   |  3 +-
 .../plugin/HiveAuthorizationValidator.java  |  6 +-
 .../authorization/plugin/HiveAuthorizer.java|  6 +-
 .../plugin/HiveAuthorizerImpl.java  |  6 +-
 .../authorization/plugin/HiveAuthzContext.java  | 99 
 .../authorization/plugin/HiveV1Authorizer.java  |  6 +-
 .../authorization/plugin/QueryContext.java  | 78 ---
 .../sqlstd/DummyHiveAuthorizationValidator.java |  8 +-
 .../SQLStdHiveAuthorizationValidator.java   |  8 +-
 .../cli/operation/MetadataOperation.java|  5 +-
 24 files changed, 150 insertions(+), 149 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/4af1be71/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
index 5e601c9..9aca713 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
@@ -425,7 +425,7 @@ public class TestHiveAuthorizerCheckInvocation {
 
 verify(mockedAuthorizer).checkPrivileges(any(HiveOperationType.class),
 inputsCapturer.capture(), outputsCapturer.capture(),
-any(QueryContext.class));
+any(HiveAuthzContext.class));
 
 return new ImmutablePair(inputsCapturer.getValue(), 
outputsCapturer.getValue());
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/4af1be71/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
index 0209044..5922a8c 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
@@ -77,7 +77,7 @@ public class TestHiveAuthorizerShowFilters {
 protected abstract class AuthorizerWithFilterCmdImpl implements 
HiveAuthorizer {
   @Override
   public List 
filterListCmdObjects(List listObjs,
-  QueryContext context) throws HiveAuthzPluginException, 
HiveAccessControlException {
+  HiveAuthzContext context) throws HiveAuthzPluginException, 
HiveAccessControlException {
 // capture arguments in static
 filterArguments = listObjs;
 // return static variable with results, if it is set to some set of
@@ -101,7 +101,7 @@ public class TestHiveAuthorizerShowFilters {
   try {
 Mockito.when(
 

hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) (addendum)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 0e5b90d51 -> 82d284a45


HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product 
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/82d284a4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/82d284a4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/82d284a4

Branch: refs/heads/branch-2.1
Commit: 82d284a45e0721cbfd6f42a2b3b39aa4eeef4337
Parents: 0e5b90d
Author: Jesus Camacho Rodriguez 
Authored: Tue May 31 23:40:21 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 23:40:21 2016 +0100

--
 .../clientpositive/constant_prop_3.q.out| 22 ++--
 .../results/clientpositive/constprog3.q.out |  4 ++--
 .../results/clientpositive/cross_join.q.out | 16 +++---
 .../test/results/clientpositive/join_view.q.out |  6 +++---
 .../clientpositive/ppd_repeated_alias.q.out |  6 +++---
 .../clientpositive/spark/join_view.q.out|  6 +++---
 .../tez/dynamic_partition_pruning.q.out | 10 -
 .../tez/dynamic_partition_pruning_2.q.out   |  4 ++--
 8 files changed, 37 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/82d284a4/ql/src/test/results/clientpositive/constant_prop_3.q.out
--
diff --git a/ql/src/test/results/clientpositive/constant_prop_3.q.out 
b/ql/src/test/results/clientpositive/constant_prop_3.q.out
index fe382fc..58f1065 100644
--- a/ql/src/test/results/clientpositive/constant_prop_3.q.out
+++ b/ql/src/test/results/clientpositive/constant_prop_3.q.out
@@ -232,7 +232,7 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col1, _col3, _col4, _col5
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column 
stats: NONE
   File Output Operator
 compressed: false
 table:
@@ -248,7 +248,7 @@ STAGE PLANS:
   key expressions: _col1 (type: int)
   sort order: +
   Map-reduce partition columns: _col1 (type: int)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: _col3 (type: string), _col4 (type: string), 
_col5 (type: int)
   TableScan
 alias: supplier_hive
@@ -273,20 +273,20 @@ STAGE PLANS:
 0 _col1 (type: int)
 1 _col0 (type: int)
   outputColumnNames: _col1, _col3, _col4, _col5, _col7
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column 
stats: NONE
   Filter Operator
 predicate: _col7 is null (type: boolean)
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column 
stats: NONE
 Select Operator
   expressions: _col3 (type: string), _col4 (type: string), _col5 
(type: int), _col1 (type: int)
   outputColumnNames: _col3, _col4, _col5, _col1
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE 
Column stats: NONE
   Group By Operator
 aggregations: count(DISTINCT _col1)
 keys: _col3 (type: string), _col4 (type: string), _col5 (type: 
int), _col1 (type: int)
 mode: hash
 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
   table:
@@ -302,14 +302,14 @@ STAGE PLANS:
   key expressions: _col0 (type: string), _col1 (type: string), 
_col2 (type: int), _col3 (type: int)
   sort order: 
   Map-reduce partition columns: _col0 (type: string), _col1 (type: 
string), _col2 (type: int)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE 
Column stats: NONE
   Reduce Operator Tree:
 Group By Operator
   aggregations: 

hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) (addendum)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/master 411c356bd -> 01c9f0875


HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product 
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan) (addendum)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/01c9f087
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/01c9f087
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/01c9f087

Branch: refs/heads/master
Commit: 01c9f08751e6c1fe534df57842fafe79213f38fb
Parents: 411c356
Author: Jesus Camacho Rodriguez 
Authored: Tue May 31 23:38:37 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 23:38:37 2016 +0100

--
 .../clientpositive/constant_prop_3.q.out| 22 ++--
 .../results/clientpositive/constprog3.q.out |  4 ++--
 .../results/clientpositive/cross_join.q.out | 16 +++---
 .../test/results/clientpositive/join_view.q.out |  6 +++---
 .../clientpositive/ppd_repeated_alias.q.out |  6 +++---
 .../clientpositive/spark/join_view.q.out|  6 +++---
 .../tez/dynamic_partition_pruning.q.out | 10 -
 .../tez/dynamic_partition_pruning_2.q.out   |  4 ++--
 8 files changed, 37 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/01c9f087/ql/src/test/results/clientpositive/constant_prop_3.q.out
--
diff --git a/ql/src/test/results/clientpositive/constant_prop_3.q.out 
b/ql/src/test/results/clientpositive/constant_prop_3.q.out
index fe382fc..58f1065 100644
--- a/ql/src/test/results/clientpositive/constant_prop_3.q.out
+++ b/ql/src/test/results/clientpositive/constant_prop_3.q.out
@@ -232,7 +232,7 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col1, _col3, _col4, _col5
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column 
stats: NONE
   File Output Operator
 compressed: false
 table:
@@ -248,7 +248,7 @@ STAGE PLANS:
   key expressions: _col1 (type: int)
   sort order: +
   Map-reduce partition columns: _col1 (type: int)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: _col3 (type: string), _col4 (type: string), 
_col5 (type: int)
   TableScan
 alias: supplier_hive
@@ -273,20 +273,20 @@ STAGE PLANS:
 0 _col1 (type: int)
 1 _col0 (type: int)
   outputColumnNames: _col1, _col3, _col4, _col5, _col7
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column 
stats: NONE
   Filter Operator
 predicate: _col7 is null (type: boolean)
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column 
stats: NONE
 Select Operator
   expressions: _col3 (type: string), _col4 (type: string), _col5 
(type: int), _col1 (type: int)
   outputColumnNames: _col3, _col4, _col5, _col1
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE 
Column stats: NONE
   Group By Operator
 aggregations: count(DISTINCT _col1)
 keys: _col3 (type: string), _col4 (type: string), _col5 (type: 
int), _col1 (type: int)
 mode: hash
 outputColumnNames: _col0, _col1, _col2, _col3, _col4
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
+Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
   table:
@@ -302,14 +302,14 @@ STAGE PLANS:
   key expressions: _col0 (type: string), _col1 (type: string), 
_col2 (type: int), _col3 (type: int)
   sort order: 
   Map-reduce partition columns: _col0 (type: string), _col1 (type: 
string), _col2 (type: int)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE 
Column stats: NONE
   Reduce Operator Tree:
 Group By Operator
   aggregations: count(DISTINCT 

hive git commit: HIVE-13839 : Refactor : remove SHIMS.getListLocatedStatus (Ashutosh Chauhan via Sergey Shelukhin)

[hashutosh]

Repository: hive
Updated Branches:
  refs/heads/master 6a01be889 -> 411c356bd


HIVE-13839 : Refactor : remove SHIMS.getListLocatedStatus (Ashutosh Chauhan via 
Sergey Shelukhin)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/411c356b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/411c356b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/411c356b

Branch: refs/heads/master
Commit: 411c356bd975231e4acf97d4ac312e4899098e30
Parents: 6a01be8
Author: Ashutosh Chauhan 
Authored: Tue May 24 16:33:34 2016 -0700
Committer: Ashutosh Chauhan 
Committed: Tue May 31 14:09:18 2016 -0700

--
 .../hcatalog/templeton/tool/TempletonUtils.java | 17 ---
 .../hive/ql/hooks/PostExecOrcFileDump.java  |  7 +++---
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |  4 ++--
 .../org/apache/hadoop/hive/ql/io/HdfsUtils.java | 23 +---
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |  4 ++--
 .../apache/hadoop/hive/shims/Hadoop23Shims.java | 17 ---
 .../apache/hadoop/hive/shims/HadoopShims.java   | 12 --
 7 files changed, 37 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/411c356b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
--
diff --git 
a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
 
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
index 83584d3..201e647 100644
--- 
a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
+++ 
b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
@@ -104,14 +104,14 @@ public class TempletonUtils {
   public static final Pattern HIVE_COMPLETE = Pattern.compile(" map = 
(\\d+%),\\s+reduce = (\\d+%).*$");
   /**
* Hive on Tez produces progress report that looks like this
-   * Map 1: -/-Reducer 2: 0/1  
-   * Map 1: -/-Reducer 2: 0(+1)/1  
+   * Map 1: -/-Reducer 2: 0/1
+   * Map 1: -/-Reducer 2: 0(+1)/1
* Map 1: -/-Reducer 2: 1/1
-   * 
+   *
* -/- means there are no tasks (yet)
* 0/1 means 1 total tasks, 0 completed
* 1(+2)/3 means 3 total, 1 completed and 2 running
-   * 
+   *
* HIVE-8495, in particular 
https://issues.apache.org/jira/secure/attachment/12675504/Screen%20Shot%202014-10-16%20at%209.35.26%20PM.png
* has more examples.
* To report progress, we'll assume all tasks are equal size and compute 
"completed" as percent of "total"
@@ -132,7 +132,7 @@ public class TempletonUtils {
 Matcher pig = PIG_COMPLETE.matcher(line);
 if (pig.find())
   return pig.group().trim();
-
+
 Matcher hive = HIVE_COMPLETE.matcher(line);
 if(hive.find()) {
   return "map " + hive.group(1) + " reduce " + hive.group(2);
@@ -274,7 +274,7 @@ public class TempletonUtils {
 if(!fs.exists(p)) {
   return Collections.emptyList();
 }
-List children = 
ShimLoader.getHadoopShims().listLocatedStatus(fs, p, null);
+FileStatus[] children = fs.listStatus(p);
 if(!isset(children)) {
   return Collections.emptyList();
 }
@@ -327,9 +327,10 @@ public class TempletonUtils {
 }
 final String finalFName = new String(fname);
 
-final FileSystem defaultFs = 
+final FileSystem defaultFs =
 ugi.doAs(new PrivilegedExceptionAction() {
-  public FileSystem run() 
+  @Override
+  public FileSystem run()
 throws URISyntaxException, IOException, InterruptedException {
 return FileSystem.get(new URI(finalFName), conf);
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/411c356b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java 
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java
index b1595ce..f1eb5cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java
@@ -30,12 +30,12 @@ import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.exec.FetchTask;
-import org.apache.orc.FileFormatException;
+import org.apache.hadoop.hive.ql.io.HdfsUtils;
 import org.apache.orc.tools.FileDump;
+import org.apache.orc.FileFormatException;
 import 

hive git commit: HIVE-13490 : Change itests to be part of the main Hive build (Zoltan Haindrich via Siddharth Seth)

[hashutosh]

Repository: hive
Updated Branches:
  refs/heads/master 8866d93b1 -> 6a01be889


HIVE-13490 : Change itests to be part of the main Hive build (Zoltan Haindrich 
via Siddharth Seth)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6a01be88
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6a01be88
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6a01be88

Branch: refs/heads/master
Commit: 6a01be889df24b07878cf99c03612d0a1ef66dba
Parents: 8866d93
Author: Ashutosh Chauhan 
Authored: Tue May 31 14:05:15 2016 -0700
Committer: Ashutosh Chauhan 
Committed: Tue May 31 14:05:15 2016 -0700

--
 pom.xml | 6 ++
 1 file changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6a01be88/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 3176caf..63a5ae1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1271,5 +1271,11 @@
 
   
 
+
+  itests
+  
+itests
+  
+
   
 

hive git commit: HIVE-12983 Provide a builtin function to get Hive version (Lenni Kuff via Szehon Ho)

[hashutosh]

Repository: hive
Updated Branches:
  refs/heads/master 1ca01701b -> a55f4a3a4


HIVE-12983 Provide a builtin function to get Hive version (Lenni Kuff via 
Szehon Ho)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a55f4a3a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a55f4a3a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a55f4a3a

Branch: refs/heads/master
Commit: a55f4a3a4b017d7f3b9279ef7d843e4b5f7fcfa0
Parents: 1ca0170
Author: Ashutosh Chauhan 
Authored: Tue May 31 13:28:44 2016 -0700
Committer: Ashutosh Chauhan 
Committed: Tue May 31 13:28:44 2016 -0700

--
 .../hadoop/hive/ql/exec/FunctionRegistry.java   |  4 ++
 .../apache/hadoop/hive/ql/udf/UDFVersion.java   | 40 
 .../hadoop/hive/ql/udf/TestUDFVersion.java  | 34 +
 .../test/queries/clientpositive/udf_version.q   |  2 +
 .../results/clientpositive/show_functions.q.out |  1 +
 .../results/clientpositive/udf_version.q.out| 11 ++
 6 files changed, 92 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/a55f4a3a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 8217ad3..69a18cd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -106,6 +106,7 @@ import org.apache.hadoop.hive.ql.udf.UDFType;
 import org.apache.hadoop.hive.ql.udf.UDFUUID;
 import org.apache.hadoop.hive.ql.udf.UDFUnbase64;
 import org.apache.hadoop.hive.ql.udf.UDFUnhex;
+import org.apache.hadoop.hive.ql.udf.UDFVersion;
 import org.apache.hadoop.hive.ql.udf.UDFWeekOfYear;
 import org.apache.hadoop.hive.ql.udf.UDFYear;
 import org.apache.hadoop.hive.ql.udf.generic.*;
@@ -350,6 +351,9 @@ public final class FunctionRegistry {
 system.registerGenericUDF("ewah_bitmap_or", GenericUDFEWAHBitmapOr.class);
 system.registerGenericUDF("ewah_bitmap_empty", 
GenericUDFEWAHBitmapEmpty.class);
 
+// Utility UDFs
+system.registerUDF("version", UDFVersion.class, false);
+
 // Aliases for Java Class Names
 // These are used in getImplicitConvertUDFMethod
 system.registerUDF(serdeConstants.BOOLEAN_TYPE_NAME, UDFToBoolean.class, 
false, UDFToBoolean.class.getSimpleName());

http://git-wip-us.apache.org/repos/asf/hive/blob/a55f4a3a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFVersion.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFVersion.java 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFVersion.java
new file mode 100644
index 000..d9750c1
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFVersion.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.Text;
+import org.apache.hive.common.util.HiveVersionInfo;
+
+
+/**
+ * UDFVersion
+ */
+@Description(name = "version",
+value="_FUNC_() - Returns the Hive build version string - includes base " +
+  "version and revision.")
+public class UDFVersion extends UDF {
+  private static final String versionInfo = String.format("%s r%s",
+  HiveVersionInfo.getVersion(), HiveVersionInfo.getRevision());
+
+  public Text evaluate() {
+ return new Text(versionInfo);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/a55f4a3a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFVersion.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFVersion.java 

hive git commit: HIVE-13518 : Hive on Tez: Shuffle joins do not choose the right 'big' table. (Vikram Dixit via Gunther Hagleitner)

[hashutosh]

Repository: hive
Updated Branches:
  refs/heads/master 57d1f3d85 -> 8866d93b1


HIVE-13518 : Hive on Tez: Shuffle joins do not choose the right 'big' table. 
(Vikram Dixit via Gunther Hagleitner)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8866d93b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8866d93b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8866d93b

Branch: refs/heads/master
Commit: 8866d93b14a3489018e6516f8b85ef1a7f18960b
Parents: 57d1f3d
Author: Ashutosh Chauhan 
Authored: Tue May 31 13:54:59 2016 -0700
Committer: Ashutosh Chauhan 
Committed: Tue May 31 13:57:51 2016 -0700

--
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   | 84 +++-
 .../apache/hadoop/hive/ql/parse/GenTezWork.java |  3 +-
 .../clientpositive/tez/metadataonly1.q.out  |  2 +-
 .../clientpositive/tez/vectorized_ptf.q.out |  2 +-
 4 files changed, 49 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/8866d93b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index b35f075..387f47d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -87,6 +87,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
 OptimizeTezProcContext context = (OptimizeTezProcContext) procCtx;
 
 JoinOperator joinOp = (JoinOperator) nd;
+long maxSize = 
context.conf.getLongVar(HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD);
 
 TezBucketJoinProcCtx tezBucketJoinProcCtx = new 
TezBucketJoinProcCtx(context.conf);
 if (!context.conf.getBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN)) {
@@ -110,7 +111,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
   numBuckets = 1;
 }
 LOG.info("Estimated number of buckets " + numBuckets);
-int mapJoinConversionPos = getMapJoinConversionPos(joinOp, context, 
numBuckets);
+int mapJoinConversionPos = getMapJoinConversionPos(joinOp, context, 
numBuckets, false, maxSize);
 if (mapJoinConversionPos < 0) {
   Object retval = checkAndConvertSMBJoin(context, joinOp, 
tezBucketJoinProcCtx);
   if (retval == null) {
@@ -134,7 +135,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
 // check if we can convert to map join no bucket scaling.
 LOG.info("Convert to non-bucketed map join");
 if (numBuckets != 1) {
-  mapJoinConversionPos = getMapJoinConversionPos(joinOp, context, 1);
+  mapJoinConversionPos = getMapJoinConversionPos(joinOp, context, 1, 
false, maxSize);
 }
 if (mapJoinConversionPos < 0) {
   // we are just converting to a common merge join operator. The shuffle
@@ -359,7 +360,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
   // MapRecordProcessor and ReduceRecordProcessor with respect to the 
sources.
   @SuppressWarnings({"rawtypes","unchecked"})
   Set set =
-  
OperatorUtils.findOperatorsUpstream((Collection)parentOp.getParentOperators(),
+  OperatorUtils.findOperatorsUpstream(parentOp.getParentOperators(),
   ReduceSinkOperator.class);
   if (size < 0) {
 size = set.size();
@@ -505,44 +506,42 @@ public class ConvertJoinMapJoin implements NodeProcessor {
   }
 
   public int getMapJoinConversionPos(JoinOperator joinOp, 
OptimizeTezProcContext context,
-  int buckets) throws SemanticException {
-/*
- * HIVE-9038: Join tests fail in tez when we have more than 1 join on the 
same key and there is
- * an outer join down the join tree that requires filterTag. We disable 
this conversion to map
- * join here now. We need to emulate the behavior of HashTableSinkOperator 
as in MR or create a
- * new operation to be able to support this. This seems like a corner case 
enough to special
- * case this for now.
- */
-if (joinOp.getConf().getConds().length > 1) {
-  boolean hasOuter = false;
-  for (JoinCondDesc joinCondDesc : joinOp.getConf().getConds()) {
-switch (joinCondDesc.getType()) {
-case JoinDesc.INNER_JOIN:
-case JoinDesc.LEFT_SEMI_JOIN:
-case JoinDesc.UNIQUE_JOIN:
-  hasOuter = false;
-  break;
-
-case JoinDesc.FULL_OUTER_JOIN:
-case JoinDesc.LEFT_OUTER_JOIN:
-case JoinDesc.RIGHT_OUTER_JOIN:
-  hasOuter = true;
-  break;
-
-default:
-  throw new 

hive git commit: HIVE-13751: LlapOutputFormatService should have a configurable send buffer size (Prasanth Jayachandran reviewed by Jason Dere)

[prasanthj]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 70e9dcddd -> b7759da73


HIVE-13751: LlapOutputFormatService should have a configurable send buffer size 
(Prasanth Jayachandran reviewed by Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b7759da7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b7759da7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b7759da7

Branch: refs/heads/branch-2.1
Commit: b7759da73c284cd3773e86fc29e14dead1ec6cee
Parents: 70e9dcd
Author: Prasanth Jayachandran 
Authored: Tue May 31 13:54:55 2016 -0700
Committer: Prasanth Jayachandran 
Committed: Tue May 31 13:55:23 2016 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  2 ++
 .../hive/llap/LlapOutputFormatService.java  | 24 +++-
 2 files changed, 20 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b7759da7/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index cad5d65..0164b75 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2874,6 +2874,8 @@ public class HiveConf extends Configuration {
 "protocol or ZK paths), similar to how ssh refuses a key with bad 
access permissions."),
 LLAP_DAEMON_OUTPUT_SERVICE_PORT("hive.llap.daemon.output.service.port", 
15003,
 "LLAP daemon output service port"),
+
LLAP_DAEMON_OUTPUT_SERVICE_SEND_BUFFER_SIZE("hive.llap.daemon.output.service.send.buffer.size",
+128 * 1024, "Send buffer size to be used by LLAP daemon output 
service"),
 LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap", 
false,
 "Override if grace join should be allowed to run in llap."),
 

http://git-wip-us.apache.org/repos/asf/hive/blob/b7759da7/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java 
b/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java
index f852041..06660b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java
@@ -109,15 +109,18 @@ public class LlapOutputFormatService {
 LOG.info("Starting LlapOutputFormatService");
 
 int portFromConf = HiveConf.getIntVar(conf, 
HiveConf.ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT);
+int sendBufferSize = HiveConf.getIntVar(conf,
+HiveConf.ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_SEND_BUFFER_SIZE);
 eventLoopGroup = new NioEventLoopGroup(1);
 serverBootstrap = new ServerBootstrap();
 serverBootstrap.group(eventLoopGroup);
 serverBootstrap.channel(NioServerSocketChannel.class);
-serverBootstrap.childHandler(new LlapOutputFormatServiceChannelHandler());
+serverBootstrap.childHandler(new 
LlapOutputFormatServiceChannelHandler(sendBufferSize));
 try {
   listeningChannelFuture = serverBootstrap.bind(portFromConf).sync();
   this.port = ((InetSocketAddress) 
listeningChannelFuture.channel().localAddress()).getPort();
-  LOG.info("LlapOutputFormatService: Binding to port " + this.port);
+  LOG.info("LlapOutputFormatService: Binding to port: {} with send buffer 
size: {} ", this.port,
+  sendBufferSize);
 } catch (InterruptedException err) {
   throw new IOException("LlapOutputFormatService: Error binding to port " 
+ portFromConf, err);
 }
@@ -154,6 +157,11 @@ public class LlapOutputFormatService {
   }
 
   protected class LlapOutputFormatServiceHandler extends 
SimpleChannelInboundHandler {
+private final int sendBufferSize;
+public LlapOutputFormatServiceHandler(final int sendBufferSize) {
+  this.sendBufferSize = sendBufferSize;
+}
+
 @Override
 public void channelRead0(ChannelHandlerContext ctx, String msg) {
   String id = msg;
@@ -162,9 +170,8 @@ public class LlapOutputFormatService {
 
 private void registerReader(ChannelHandlerContext ctx, String id) {
   synchronized(INSTANCE) {
-LOG.debug("registering socket for: "+id);
-int bufSize = 128 * 1024; // configable?
-OutputStream stream = new ChannelOutputStream(ctx, id, bufSize);
+LOG.debug("registering socket for: " + id);
+OutputStream stream = new ChannelOutputStream(ctx, id, sendBufferSize);
 LlapRecordWriter writer = new LlapRecordWriter(stream);
 writers.put(id, writer);
 
@@ 

[1/2] hive git commit: HIVE-13444 : LLAP: add HMAC signatures to LLAP; verify them on LLAP side (Sergey Shelukhin, reviewed by Siddharth Seth)

[sershe]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 44a8f0a9c -> 9fe3dab7f
  refs/heads/master 3eeb94229 -> 1ca01701b


HIVE-13444 : LLAP: add HMAC signatures to LLAP; verify them on LLAP side 
(Sergey Shelukhin, reviewed by Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1ca01701
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1ca01701
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1ca01701

Branch: refs/heads/master
Commit: 1ca01701bc8077659a9c37b849d6ac5e48c8ae72
Parents: 3eeb942
Author: Sergey Shelukhin 
Authored: Tue May 31 13:09:45 2016 -0700
Committer: Sergey Shelukhin 
Committed: Tue May 31 13:23:59 2016 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  11 +-
 .../hive/llap/security/LlapTokenClient.java | 148 ++
 .../llap/security/LlapTokenClientFactory.java   | 160 ---
 .../llap/security/LlapTokenLocalClient.java |  12 +-
 .../hadoop/hive/llap/security/LlapSigner.java   |  41 
 .../hive/llap/security/LlapTokenIdentifier.java |  14 +-
 .../hive/llap/security/SecretManager.java   |  96 ++---
 .../llap/security/SigningSecretManager.java |  26 +++
 .../llap/daemon/impl/ContainerRunnerImpl.java   |  57 +-
 .../hive/llap/daemon/impl/LlapDaemon.java   |   2 +-
 .../daemon/impl/LlapProtocolServerImpl.java |  59 --
 .../hive/llap/daemon/impl/LlapTokenChecker.java |  30 ++-
 .../hive/llap/daemon/impl/QueryTracker.java |  14 +-
 .../llap/daemon/impl/TaskExecutorService.java   |   6 +-
 .../llap/daemon/impl/TaskRunnerCallable.java|  19 +-
 .../hive/llap/security/LlapSignerImpl.java  |  60 ++
 .../daemon/impl/TaskExecutorTestHelpers.java|   3 +-
 .../llap/daemon/impl/TestLlapTokenChecker.java  |   8 +-
 .../TestFirstInFirstOutComparator.java  |  31 ---
 .../hive/llap/security/TestLlapSignerImpl.java  | 200 +++
 .../hive/ql/exec/tez/TezSessionState.java   |  26 +--
 21 files changed, 720 insertions(+), 303 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1ca01701/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index cdff4db..8706665 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2716,6 +2716,12 @@ public class HiveConf extends Configuration {
 LLAP_MANAGEMENT_ACL("hive.llap.management.acl", "*", "The ACL for LLAP 
daemon management."),
 LLAP_MANAGEMENT_ACL_DENY("hive.llap.management.acl.blocked", "",
 "The deny ACL for LLAP daemon management."),
+
LLAP_REMOTE_TOKEN_REQUIRES_SIGNING("hive.llap.remote.token.requires.signing", 
"true",
+new StringSet("false", "except_llap_owner", "true"),
+"Whether the token returned from LLAP management API should require 
fragment signing.\n" +
+"True by default; can be disabled to allow CLI to get tokens from LLAP 
in a secure\n" +
+"cluster by setting it to true or 'except_llap_owner' (the latter 
returns such tokens\n" +
+"to everyone except the user LLAP cluster is authenticating under)."),
 
 // Hadoop DelegationTokenManager default is 1 week.
 
LLAP_DELEGATION_TOKEN_LIFETIME("hive.llap.daemon.delegation.token.lifetime", 
"14d",
@@ -2725,11 +2731,6 @@ public class HiveConf extends Configuration {
 "RPC port for LLAP daemon management service."),
 LLAP_WEB_AUTO_AUTH("hive.llap.auto.auth", false,
 "Whether or not to set Hadoop configs to enable auth in LLAP web 
app."),
-LLAP_CREATE_TOKEN_LOCALLY("hive.llap.create.token.locally", "hs2",
-new StringSet("true", "hs2", "false"),
-"Whether to create LLAP tokens locally, saving directly to ZooKeeper 
SecretManager.\n" +
-"Requires one to have access to ZK paths; in other words, this should 
only be used in\n" +
-"HiveServer2. By default, the value is 'hs2', which means exactly 
that."),
 
 LLAP_DAEMON_RPC_NUM_HANDLERS("hive.llap.daemon.rpc.num.handlers", 5,
   "Number of RPC handlers for LLAP daemon.", 
"llap.daemon.rpc.num.handlers"),

http://git-wip-us.apache.org/repos/asf/hive/blob/1ca01701/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenClient.java
--
diff --git 
a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenClient.java
 
b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenClient.java
new file mode 100644
index 000..921e050
--- /dev/null
+++ 

[2/2] hive git commit: HIVE-13444 : LLAP: add HMAC signatures to LLAP; verify them on LLAP side (Sergey Shelukhin, reviewed by Siddharth Seth)

[sershe]

HIVE-13444 : LLAP: add HMAC signatures to LLAP; verify them on LLAP side 
(Sergey Shelukhin, reviewed by Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9fe3dab7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9fe3dab7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9fe3dab7

Branch: refs/heads/branch-2.1
Commit: 9fe3dab7fe82d78435d9cd01f44f7a8e748f3420
Parents: 44a8f0a
Author: Sergey Shelukhin 
Authored: Tue May 31 13:09:45 2016 -0700
Committer: Sergey Shelukhin 
Committed: Tue May 31 13:24:05 2016 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  11 +-
 .../hive/llap/security/LlapTokenClient.java | 148 ++
 .../llap/security/LlapTokenClientFactory.java   | 160 ---
 .../llap/security/LlapTokenLocalClient.java |  12 +-
 .../hadoop/hive/llap/security/LlapSigner.java   |  41 
 .../hive/llap/security/LlapTokenIdentifier.java |  14 +-
 .../hive/llap/security/SecretManager.java   |  96 ++---
 .../llap/security/SigningSecretManager.java |  26 +++
 .../llap/daemon/impl/ContainerRunnerImpl.java   |  57 +-
 .../hive/llap/daemon/impl/LlapDaemon.java   |   2 +-
 .../daemon/impl/LlapProtocolServerImpl.java |  59 --
 .../hive/llap/daemon/impl/LlapTokenChecker.java |  30 ++-
 .../hive/llap/daemon/impl/QueryTracker.java |  14 +-
 .../llap/daemon/impl/TaskExecutorService.java   |   6 +-
 .../llap/daemon/impl/TaskRunnerCallable.java|  19 +-
 .../hive/llap/security/LlapSignerImpl.java  |  60 ++
 .../daemon/impl/TaskExecutorTestHelpers.java|   3 +-
 .../llap/daemon/impl/TestLlapTokenChecker.java  |   8 +-
 .../TestFirstInFirstOutComparator.java  |  31 ---
 .../hive/llap/security/TestLlapSignerImpl.java  | 200 +++
 .../hive/ql/exec/tez/TezSessionState.java   |  26 +--
 21 files changed, 720 insertions(+), 303 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/9fe3dab7/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 6a404bd..cad5d65 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2716,6 +2716,12 @@ public class HiveConf extends Configuration {
 LLAP_MANAGEMENT_ACL("hive.llap.management.acl", "*", "The ACL for LLAP 
daemon management."),
 LLAP_MANAGEMENT_ACL_DENY("hive.llap.management.acl.blocked", "",
 "The deny ACL for LLAP daemon management."),
+
LLAP_REMOTE_TOKEN_REQUIRES_SIGNING("hive.llap.remote.token.requires.signing", 
"true",
+new StringSet("false", "except_llap_owner", "true"),
+"Whether the token returned from LLAP management API should require 
fragment signing.\n" +
+"True by default; can be disabled to allow CLI to get tokens from LLAP 
in a secure\n" +
+"cluster by setting it to true or 'except_llap_owner' (the latter 
returns such tokens\n" +
+"to everyone except the user LLAP cluster is authenticating under)."),
 
 // Hadoop DelegationTokenManager default is 1 week.
 
LLAP_DELEGATION_TOKEN_LIFETIME("hive.llap.daemon.delegation.token.lifetime", 
"14d",
@@ -2725,11 +2731,6 @@ public class HiveConf extends Configuration {
 "RPC port for LLAP daemon management service."),
 LLAP_WEB_AUTO_AUTH("hive.llap.auto.auth", false,
 "Whether or not to set Hadoop configs to enable auth in LLAP web 
app."),
-LLAP_CREATE_TOKEN_LOCALLY("hive.llap.create.token.locally", "hs2",
-new StringSet("true", "hs2", "false"),
-"Whether to create LLAP tokens locally, saving directly to ZooKeeper 
SecretManager.\n" +
-"Requires one to have access to ZK paths; in other words, this should 
only be used in\n" +
-"HiveServer2. By default, the value is 'hs2', which means exactly 
that."),
 
 LLAP_DAEMON_RPC_NUM_HANDLERS("hive.llap.daemon.rpc.num.handlers", 5,
   "Number of RPC handlers for LLAP daemon.", 
"llap.daemon.rpc.num.handlers"),

http://git-wip-us.apache.org/repos/asf/hive/blob/9fe3dab7/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenClient.java
--
diff --git 
a/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenClient.java
 
b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenClient.java
new file mode 100644
index 000..921e050
--- /dev/null
+++ 
b/llap-client/src/java/org/apache/hadoop/hive/llap/security/LlapTokenClient.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache 

hive git commit: HIVE-13852: NPE in TaskLocationHints during LLAP GetSplits request (Jason Dere, reviewed by Siddharth Seth)

[jdere]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 545973cdf -> 44a8f0a9c


HIVE-13852: NPE in TaskLocationHints during LLAP GetSplits request (Jason Dere, 
reviewed by Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/44a8f0a9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/44a8f0a9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/44a8f0a9

Branch: refs/heads/branch-2.1
Commit: 44a8f0a9c30e0fda402eee2b533e159580978386
Parents: 545973c
Author: Jason Dere 
Authored: Tue May 31 12:21:28 2016 -0700
Committer: Jason Dere 
Committed: Tue May 31 12:30:11 2016 -0700

--
 .../apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java   | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/44a8f0a9/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
index 83d492a..140dbda 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
@@ -91,7 +91,9 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.TaskLocationHint;
 import org.apache.tez.dag.api.TaskSpecBuilder;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.mapreduce.grouper.TezSplitGrouper;
 import org.apache.tez.runtime.api.Event;
 import org.apache.tez.runtime.api.events.InputConfigureVertexTasksEvent;
 import org.apache.tez.runtime.api.impl.EventMetaData;
@@ -205,6 +207,7 @@ public class GenericUDTFGetSplits extends GenericUDTF {
 HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE, "llap");
 HiveConf.setBoolVar(conf, 
HiveConf.ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS, true);
 HiveConf.setBoolVar(conf, HiveConf.ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS, 
true);
+conf.setBoolean(TezSplitGrouper.TEZ_GROUPING_NODE_LOCAL_ONLY, true);
 // Tez/LLAP requires RPC query plan
 HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_RPC_QUERY_PLAN, true);
 

hive git commit: HIVE-13852: NPE in TaskLocationHints during LLAP GetSplits request (Jason Dere, reviewed by Siddharth Seth)

[jdere]

Repository: hive
Updated Branches:
  refs/heads/master acdc31b8d -> 3eeb94229


HIVE-13852: NPE in TaskLocationHints during LLAP GetSplits request (Jason Dere, 
reviewed by Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3eeb9422
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3eeb9422
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3eeb9422

Branch: refs/heads/master
Commit: 3eeb94229af5577e945cb3f5080b83948296ef0c
Parents: acdc31b
Author: Jason Dere 
Authored: Tue May 31 12:21:28 2016 -0700
Committer: Jason Dere 
Committed: Tue May 31 12:23:33 2016 -0700

--
 .../apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java   | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/3eeb9422/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
index 83d492a..140dbda 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
@@ -91,7 +91,9 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.TaskLocationHint;
 import org.apache.tez.dag.api.TaskSpecBuilder;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.mapreduce.grouper.TezSplitGrouper;
 import org.apache.tez.runtime.api.Event;
 import org.apache.tez.runtime.api.events.InputConfigureVertexTasksEvent;
 import org.apache.tez.runtime.api.impl.EventMetaData;
@@ -205,6 +207,7 @@ public class GenericUDTFGetSplits extends GenericUDTF {
 HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_MODE, "llap");
 HiveConf.setBoolVar(conf, 
HiveConf.ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS, true);
 HiveConf.setBoolVar(conf, HiveConf.ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS, 
true);
+conf.setBoolean(TezSplitGrouper.TEZ_GROUPING_NODE_LOCAL_ONLY, true);
 // Tez/LLAP requires RPC query plan
 HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_RPC_QUERY_PLAN, true);
 

[2/3] hive git commit: HIVE-13870 : Decimal vector is not resized correctly (Sergey Shelukhin, reviewed by Matt McCline)

[sershe]

HIVE-13870 : Decimal vector is not resized correctly (Sergey Shelukhin, 
reviewed by Matt McCline)

Conflicts:

storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f0e07203
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f0e07203
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f0e07203

Branch: refs/heads/branch-2.0
Commit: f0e072030062210b3927b9ffbbec97d8e1d66e4d
Parents: e3cfeeb
Author: Sergey Shelukhin 
Authored: Tue May 31 11:48:44 2016 -0700
Committer: Sergey Shelukhin 
Committed: Tue May 31 11:58:54 2016 -0700

--
 .../ql/exec/vector/DecimalColumnVector.java | 32 
 1 file changed, 13 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f0e07203/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
--
diff --git 
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
 
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
index 1523ff6..2488631 100644
--- 
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
+++ 
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
@@ -60,14 +60,6 @@ public class DecimalColumnVector extends ColumnVector {
 }
   }
 
-  // Fill the column vector with nulls
-  public void fillWithNulls() {
-noNulls = false;
-isRepeating = true;
-vector[0] = null;
-isNull[0] = true;
-  }
-
   @Override
   public void flatten(boolean selectedInUse, int[] sel, int size) {
 // TODO Auto-generated method stub
@@ -140,17 +132,19 @@ public class DecimalColumnVector extends ColumnVector {
 
   @Override
   public void ensureSize(int size, boolean preserveData) {
-if (size > vector.length) {
-  super.ensureSize(size, preserveData);
-  HiveDecimalWritable[] oldArray = vector;
-  vector = new HiveDecimalWritable[size];
-  if (preserveData) {
-// we copy all of the values to avoid creating more objects
-System.arraycopy(oldArray, 0, vector, 0 , oldArray.length);
-for(int i= oldArray.length; i < vector.length; ++i) {
-  vector[i] = new HiveDecimalWritable(HiveDecimal.ZERO);
-}
-  }
+super.ensureSize(size, preserveData);
+if (size <= vector.length) return; // We assume the existing vector is 
always valid.
+HiveDecimalWritable[] oldArray = vector;
+vector = new HiveDecimalWritable[size];
+int initPos = 0;
+if (preserveData) {
+  // we copy all of the values to avoid creating more objects
+  // TODO: it might be cheaper to always preserve data or reset existing 
objects
+  initPos = oldArray.length;
+  System.arraycopy(oldArray, 0, vector, 0 , oldArray.length);
+}
+for (int i = initPos; i < vector.length; ++i) {
+  vector[i] = new HiveDecimalWritable(HiveDecimal.ZERO);
 }
   }
 }

[1/3] hive git commit: HIVE-13870 : Decimal vector is not resized correctly (Sergey Shelukhin, reviewed by Matt McCline)

[sershe]

Repository: hive
Updated Branches:
  refs/heads/branch-2.0 e3cfeebce -> f0e072030
  refs/heads/branch-2.1 23ca1b46e -> 545973cdf
  refs/heads/master b7166d7d5 -> acdc31b8d


HIVE-13870 : Decimal vector is not resized correctly (Sergey Shelukhin, 
reviewed by Matt McCline)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/545973cd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/545973cd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/545973cd

Branch: refs/heads/branch-2.1
Commit: 545973cdfdbc622b0196113e96c391b714d6932b
Parents: 23ca1b4
Author: Sergey Shelukhin 
Authored: Tue May 31 11:48:44 2016 -0700
Committer: Sergey Shelukhin 
Committed: Tue May 31 11:56:57 2016 -0700

--
 .../ql/exec/vector/DecimalColumnVector.java | 30 
 1 file changed, 12 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/545973cd/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
--
diff --git 
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
 
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
index 0c52210..2488631 100644
--- 
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
+++ 
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
@@ -60,14 +60,6 @@ public class DecimalColumnVector extends ColumnVector {
 }
   }
 
-  // Fill the column vector with nulls
-  public void fillWithNulls() {
-noNulls = false;
-isRepeating = true;
-vector[0] = null;
-isNull[0] = true;
-  }
-
   @Override
   public void flatten(boolean selectedInUse, int[] sel, int size) {
 // TODO Auto-generated method stub
@@ -141,16 +133,18 @@ public class DecimalColumnVector extends ColumnVector {
   @Override
   public void ensureSize(int size, boolean preserveData) {
 super.ensureSize(size, preserveData);
-if (size > vector.length) {
-  HiveDecimalWritable[] oldArray = vector;
-  vector = new HiveDecimalWritable[size];
-  if (preserveData) {
-// we copy all of the values to avoid creating more objects
-System.arraycopy(oldArray, 0, vector, 0 , oldArray.length);
-for(int i= oldArray.length; i < vector.length; ++i) {
-  vector[i] = new HiveDecimalWritable(HiveDecimal.ZERO);
-}
-  }
+if (size <= vector.length) return; // We assume the existing vector is 
always valid.
+HiveDecimalWritable[] oldArray = vector;
+vector = new HiveDecimalWritable[size];
+int initPos = 0;
+if (preserveData) {
+  // we copy all of the values to avoid creating more objects
+  // TODO: it might be cheaper to always preserve data or reset existing 
objects
+  initPos = oldArray.length;
+  System.arraycopy(oldArray, 0, vector, 0 , oldArray.length);
+}
+for (int i = initPos; i < vector.length; ++i) {
+  vector[i] = new HiveDecimalWritable(HiveDecimal.ZERO);
 }
   }
 }

[3/3] hive git commit: HIVE-13870 : Decimal vector is not resized correctly (Sergey Shelukhin, reviewed by Matt McCline)

[sershe]

HIVE-13870 : Decimal vector is not resized correctly (Sergey Shelukhin, 
reviewed by Matt McCline)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/acdc31b8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/acdc31b8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/acdc31b8

Branch: refs/heads/master
Commit: acdc31b8d3c2b85e4b7882422f5a35cf8665eb8d
Parents: b7166d7
Author: Sergey Shelukhin 
Authored: Tue May 31 11:48:44 2016 -0700
Committer: Sergey Shelukhin 
Committed: Tue May 31 11:59:09 2016 -0700

--
 .../ql/exec/vector/DecimalColumnVector.java | 30 
 1 file changed, 12 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/acdc31b8/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
--
diff --git 
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
 
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
index 0c52210..2488631 100644
--- 
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
+++ 
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
@@ -60,14 +60,6 @@ public class DecimalColumnVector extends ColumnVector {
 }
   }
 
-  // Fill the column vector with nulls
-  public void fillWithNulls() {
-noNulls = false;
-isRepeating = true;
-vector[0] = null;
-isNull[0] = true;
-  }
-
   @Override
   public void flatten(boolean selectedInUse, int[] sel, int size) {
 // TODO Auto-generated method stub
@@ -141,16 +133,18 @@ public class DecimalColumnVector extends ColumnVector {
   @Override
   public void ensureSize(int size, boolean preserveData) {
 super.ensureSize(size, preserveData);
-if (size > vector.length) {
-  HiveDecimalWritable[] oldArray = vector;
-  vector = new HiveDecimalWritable[size];
-  if (preserveData) {
-// we copy all of the values to avoid creating more objects
-System.arraycopy(oldArray, 0, vector, 0 , oldArray.length);
-for(int i= oldArray.length; i < vector.length; ++i) {
-  vector[i] = new HiveDecimalWritable(HiveDecimal.ZERO);
-}
-  }
+if (size <= vector.length) return; // We assume the existing vector is 
always valid.
+HiveDecimalWritable[] oldArray = vector;
+vector = new HiveDecimalWritable[size];
+int initPos = 0;
+if (preserveData) {
+  // we copy all of the values to avoid creating more objects
+  // TODO: it might be cheaper to always preserve data or reset existing 
objects
+  initPos = oldArray.length;
+  System.arraycopy(oldArray, 0, vector, 0 , oldArray.length);
+}
+for (int i = initPos; i < vector.length; ++i) {
+  vector[i] = new HiveDecimalWritable(HiveDecimal.ZERO);
 }
   }
 }

hive git commit: HIVE-13840: Orc split generation is reading file footers twice (Prasanth Jayachandran reviewed by Owen O'Malley)

[prasanthj]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 2068e3c0c -> 23ca1b46e


HIVE-13840: Orc split generation is reading file footers twice (Prasanth 
Jayachandran reviewed by Owen O'Malley)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/23ca1b46
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/23ca1b46
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/23ca1b46

Branch: refs/heads/branch-2.1
Commit: 23ca1b46e78759750b4726bedd97fef7c632e888
Parents: 2068e3c
Author: Prasanth Jayachandran 
Authored: Tue May 31 11:48:16 2016 -0700
Committer: Prasanth Jayachandran 
Committed: Tue May 31 11:50:05 2016 -0700

--
 .../java/org/apache/orc/impl/ReaderImpl.java|   1 +
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |   4 +-
 .../hadoop/hive/ql/io/orc/ReaderImpl.java   |   9 +-
 .../hive/ql/io/orc/TestInputOutputFormat.java   | 159 +++
 4 files changed, 165 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/23ca1b46/orc/src/java/org/apache/orc/impl/ReaderImpl.java
--
diff --git a/orc/src/java/org/apache/orc/impl/ReaderImpl.java 
b/orc/src/java/org/apache/orc/impl/ReaderImpl.java
index 2da590e..1dd5e43 100644
--- a/orc/src/java/org/apache/orc/impl/ReaderImpl.java
+++ b/orc/src/java/org/apache/orc/impl/ReaderImpl.java
@@ -345,6 +345,7 @@ public class ReaderImpl implements Reader {
 options.getMaxLength());
 this.footerMetaAndPsBuffer = footerMetaData.footerMetaAndPsBuffer;
   }
+  options.fileMetaInfo(footerMetaData);
   MetaInfoObjExtractor rInfo =
   new MetaInfoObjExtractor(footerMetaData.compressionType,
footerMetaData.bufferSize,

http://git-wip-us.apache.org/repos/asf/hive/blob/23ca1b46/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 087207b..185852c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -467,7 +467,7 @@ public class OrcInputFormat implements 
InputFormat,
   }
   try {
 OrcFile.createReader(file.getPath(),
-OrcFile.readerOptions(conf).filesystem(fs));
+
OrcFile.readerOptions(conf).filesystem(fs).maxLength(file.getLen()));
   } catch (IOException e) {
 return false;
   }
@@ -1391,7 +1391,7 @@ public class OrcInputFormat implements 
InputFormat,
 
 private Reader createOrcReader() throws IOException {
   return OrcFile.createReader(file.getPath(),
-  OrcFile.readerOptions(context.conf).filesystem(fs));
+  
OrcFile.readerOptions(context.conf).filesystem(fs).maxLength(file.getLen()));
 }
 
 private long computeProjectionSize(List types,

http://git-wip-us.apache.org/repos/asf/hive/blob/23ca1b46/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index 3a2e7d8..0b40fef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -56,10 +56,10 @@ public class ReaderImpl extends 
org.apache.orc.impl.ReaderImpl
   //serialized footer - Keeping this around for use by getFileMetaInfo()
   // will help avoid cpu cycles spend in deserializing at cost of increased
   // memory footprint.
-  private final ByteBuffer footerByteBuffer;
+  private ByteBuffer footerByteBuffer;
   // Same for metastore cache - maintains the same background buffer, but 
includes postscript.
   // This will only be set if the file footer/metadata was read from disk.
-  private final ByteBuffer footerMetaAndPsBuffer;
+  private ByteBuffer footerMetaAndPsBuffer;
 
   @Override
   public ObjectInspector getObjectInspector() {
@@ -89,18 +89,15 @@ public class ReaderImpl extends 
org.apache.orc.impl.ReaderImpl
 FileMetadata fileMetadata = options.getFileMetadata();
 if (fileMetadata != null) {
   this.inspector =  OrcStruct.createObjectInspector(0, 
fileMetadata.getTypes());
-  this.footerByteBuffer = null; // not cached and not needed here
-  this.footerMetaAndPsBuffer = null;
 } else {
   FileMetaInfo footerMetaData;
   if (options.getFileMetaInfo() != null) {
 

hive git commit: HIVE-13840: Orc split generation is reading file footers twice (Prasanth Jayachandran reviewed by Owen O'Malley)

[prasanthj]

Repository: hive
Updated Branches:
  refs/heads/master e1626ef3a -> b7166d7d5


HIVE-13840: Orc split generation is reading file footers twice (Prasanth 
Jayachandran reviewed by Owen O'Malley)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b7166d7d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b7166d7d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b7166d7d

Branch: refs/heads/master
Commit: b7166d7d5d5fc28c88cc245ef74b48ded20da24a
Parents: e1626ef
Author: Prasanth Jayachandran 
Authored: Tue May 31 11:48:16 2016 -0700
Committer: Prasanth Jayachandran 
Committed: Tue May 31 11:49:27 2016 -0700

--
 .../java/org/apache/orc/impl/ReaderImpl.java|   1 +
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |   4 +-
 .../hadoop/hive/ql/io/orc/ReaderImpl.java   |   9 +-
 .../hive/ql/io/orc/TestInputOutputFormat.java   | 159 +++
 4 files changed, 165 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b7166d7d/orc/src/java/org/apache/orc/impl/ReaderImpl.java
--
diff --git a/orc/src/java/org/apache/orc/impl/ReaderImpl.java 
b/orc/src/java/org/apache/orc/impl/ReaderImpl.java
index 2da590e..1dd5e43 100644
--- a/orc/src/java/org/apache/orc/impl/ReaderImpl.java
+++ b/orc/src/java/org/apache/orc/impl/ReaderImpl.java
@@ -345,6 +345,7 @@ public class ReaderImpl implements Reader {
 options.getMaxLength());
 this.footerMetaAndPsBuffer = footerMetaData.footerMetaAndPsBuffer;
   }
+  options.fileMetaInfo(footerMetaData);
   MetaInfoObjExtractor rInfo =
   new MetaInfoObjExtractor(footerMetaData.compressionType,
footerMetaData.bufferSize,

http://git-wip-us.apache.org/repos/asf/hive/blob/b7166d7d/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 087207b..185852c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -467,7 +467,7 @@ public class OrcInputFormat implements 
InputFormat,
   }
   try {
 OrcFile.createReader(file.getPath(),
-OrcFile.readerOptions(conf).filesystem(fs));
+
OrcFile.readerOptions(conf).filesystem(fs).maxLength(file.getLen()));
   } catch (IOException e) {
 return false;
   }
@@ -1391,7 +1391,7 @@ public class OrcInputFormat implements 
InputFormat,
 
 private Reader createOrcReader() throws IOException {
   return OrcFile.createReader(file.getPath(),
-  OrcFile.readerOptions(context.conf).filesystem(fs));
+  
OrcFile.readerOptions(context.conf).filesystem(fs).maxLength(file.getLen()));
 }
 
 private long computeProjectionSize(List types,

http://git-wip-us.apache.org/repos/asf/hive/blob/b7166d7d/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java 
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
index 3a2e7d8..0b40fef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
@@ -56,10 +56,10 @@ public class ReaderImpl extends 
org.apache.orc.impl.ReaderImpl
   //serialized footer - Keeping this around for use by getFileMetaInfo()
   // will help avoid cpu cycles spend in deserializing at cost of increased
   // memory footprint.
-  private final ByteBuffer footerByteBuffer;
+  private ByteBuffer footerByteBuffer;
   // Same for metastore cache - maintains the same background buffer, but 
includes postscript.
   // This will only be set if the file footer/metadata was read from disk.
-  private final ByteBuffer footerMetaAndPsBuffer;
+  private ByteBuffer footerMetaAndPsBuffer;
 
   @Override
   public ObjectInspector getObjectInspector() {
@@ -89,18 +89,15 @@ public class ReaderImpl extends 
org.apache.orc.impl.ReaderImpl
 FileMetadata fileMetadata = options.getFileMetadata();
 if (fileMetadata != null) {
   this.inspector =  OrcStruct.createObjectInspector(0, 
fileMetadata.getTypes());
-  this.footerByteBuffer = null; // not cached and not needed here
-  this.footerMetaAndPsBuffer = null;
 } else {
   FileMetaInfo footerMetaData;
   if (options.getFileMetaInfo() != null) {
 footerMetaData = 

hive git commit: HIVE-13719. Fix failing test TestConverters. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit e1626ef3aeecb1ccf7acf4056fb7bcf23243be25)

[sseth]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 74c81a892 -> 2068e3c0c


HIVE-13719. Fix failing test TestConverters. (Siddharth Seth, reviewed by 
Sergey Shelukhin)
(cherry picked from commit e1626ef3aeecb1ccf7acf4056fb7bcf23243be25)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2068e3c0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2068e3c0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2068e3c0

Branch: refs/heads/branch-2.1
Commit: 2068e3c0ca429ed90b122970a14765eb81bc277e
Parents: 74c81a8
Author: Siddharth Seth 
Authored: Tue May 31 11:45:53 2016 -0700
Committer: Siddharth Seth 
Committed: Tue May 31 11:46:52 2016 -0700

--
 .../test/org/apache/hadoop/hive/llap/tez/TestConverters.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/2068e3c0/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
--
diff --git 
a/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java 
b/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
index 349ee14..1df6df0 100644
--- a/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
+++ b/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
@@ -42,7 +42,7 @@ import org.junit.Test;
 
 public class TestConverters {
 
-  @Test(timeout = 5000)
+  @Test(timeout = 1)
   public void testTaskSpecToFragmentSpec() {
 ByteBuffer procBb = ByteBuffer.allocate(4);
 procBb.putInt(0, 200);
@@ -98,7 +98,7 @@ public class TestConverters {
 
   }
 
-  @Test (timeout = 5000)
+  @Test (timeout = 1)
   public void testFragmentSpecToTaskSpec() {
 
 ByteBuffer procBb = ByteBuffer.allocate(4);
@@ -142,7 +142,7 @@ public class TestConverters {
 
 SignableVertexSpec vertexProto = builder.build();
 
-TaskSpec taskSpec = Converters.getTaskSpecfromProto(vertexProto, 0, 0, 
null);
+TaskSpec taskSpec = Converters.getTaskSpecfromProto(vertexProto, 0, 0, 
tezTaskAttemptId);
 
 assertEquals("dagName", taskSpec.getDAGName());
 assertEquals("vertexName", taskSpec.getVertexName());

hive git commit: HIVE-13719. Fix failing test TestConverters. (Siddharth Seth, reviewed by Sergey Shelukhin)

[sseth]

Repository: hive
Updated Branches:
  refs/heads/master a354507cc -> e1626ef3a


HIVE-13719. Fix failing test TestConverters. (Siddharth Seth, reviewed by 
Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e1626ef3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e1626ef3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e1626ef3

Branch: refs/heads/master
Commit: e1626ef3aeecb1ccf7acf4056fb7bcf23243be25
Parents: a354507
Author: Siddharth Seth 
Authored: Tue May 31 11:45:53 2016 -0700
Committer: Siddharth Seth 
Committed: Tue May 31 11:45:53 2016 -0700

--
 .../test/org/apache/hadoop/hive/llap/tez/TestConverters.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e1626ef3/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
--
diff --git 
a/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java 
b/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
index 349ee14..1df6df0 100644
--- a/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
+++ b/llap-common/src/test/org/apache/hadoop/hive/llap/tez/TestConverters.java
@@ -42,7 +42,7 @@ import org.junit.Test;
 
 public class TestConverters {
 
-  @Test(timeout = 5000)
+  @Test(timeout = 1)
   public void testTaskSpecToFragmentSpec() {
 ByteBuffer procBb = ByteBuffer.allocate(4);
 procBb.putInt(0, 200);
@@ -98,7 +98,7 @@ public class TestConverters {
 
   }
 
-  @Test (timeout = 5000)
+  @Test (timeout = 1)
   public void testFragmentSpecToTaskSpec() {
 
 ByteBuffer procBb = ByteBuffer.allocate(4);
@@ -142,7 +142,7 @@ public class TestConverters {
 
 SignableVertexSpec vertexProto = builder.build();
 
-TaskSpec taskSpec = Converters.getTaskSpecfromProto(vertexProto, 0, 0, 
null);
+TaskSpec taskSpec = Converters.getTaskSpecfromProto(vertexProto, 0, 0, 
tezTaskAttemptId);
 
 assertEquals("dagName", taskSpec.getDAGName());
 assertEquals("vertexName", taskSpec.getVertexName());

[2/2] hive git commit: HIVE-13859: mask() UDF not retaining day and month field values (Madhan Neethiraj via Ashutosh Chauhan)

[hashutosh]

HIVE-13859: mask() UDF not retaining day and month field values (Madhan 
Neethiraj via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/74c81a89
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/74c81a89
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/74c81a89

Branch: refs/heads/branch-2.1
Commit: 74c81a892f9e16ff0543c5284f46d99c22c55ddb
Parents: 549fc26
Author: Madhan Neethiraj 
Authored: Sat May 28 03:16:38 2016 -0700
Committer: Ashutosh Chauhan 
Committed: Tue May 31 11:41:53 2016 -0700

--
 .../hive/ql/udf/generic/GenericUDFMask.java | 12 
 ql/src/test/queries/clientpositive/udf_mask.q   |  9 -
 .../test/results/clientpositive/udf_mask.q.out  | 20 +---
 3 files changed, 33 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/74c81a89/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java
index cd35142..b815aae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java
@@ -96,12 +96,16 @@ class MaskTransformer extends AbstractTransformer {
   maskedNumber = MASKED_NUMBER;
 }
 
-if(maskedDayValue < 1 || maskedDayValue > 31) {
-  maskedDayValue = MASKED_DAY_COMPONENT_VAL;
+if(maskedDayValue != UNMASKED_VAL) {
+  if(maskedDayValue < 1 || maskedDayValue > 31) {
+maskedDayValue = MASKED_DAY_COMPONENT_VAL;
+  }
 }
 
-if(maskedMonthValue < 0 || maskedMonthValue > 11) {
-  maskedMonthValue = MASKED_MONTH_COMPONENT_VAL;
+if(maskedMonthValue != UNMASKED_VAL) {
+  if(maskedMonthValue < 0 || maskedMonthValue > 11) {
+maskedMonthValue = MASKED_MONTH_COMPONENT_VAL;
+  }
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/74c81a89/ql/src/test/queries/clientpositive/udf_mask.q
--
diff --git a/ql/src/test/queries/clientpositive/udf_mask.q 
b/ql/src/test/queries/clientpositive/udf_mask.q
index 82b8ee7..15f7d27 100644
--- a/ql/src/test/queries/clientpositive/udf_mask.q
+++ b/ql/src/test/queries/clientpositive/udf_mask.q
@@ -10,4 +10,11 @@ select mask('TestString-123', 'X', 'x', '0', ':'),
mask(cast(12345 as smallint), -1, -1, -1, -1, '5'),
mask(cast(12345 as int), -1, -1, -1, -1, '5'),
mask(cast(12345 as bigint), -1, -1, -1, -1, '5'),
-   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 0, 0, 0);
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 0, 0, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, 0, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, -1, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, 0, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, -1, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, 0, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, -1, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, -1, -1);

http://git-wip-us.apache.org/repos/asf/hive/blob/74c81a89/ql/src/test/results/clientpositive/udf_mask.q.out
--
diff --git a/ql/src/test/results/clientpositive/udf_mask.q.out 
b/ql/src/test/results/clientpositive/udf_mask.q.out
index cbafaee..ac0e273 100644
--- a/ql/src/test/results/clientpositive/udf_mask.q.out
+++ b/ql/src/test/results/clientpositive/udf_mask.q.out
@@ -53,7 +53,14 @@ PREHOOK: query: select mask('TestString-123', 'X', 'x', '0', 
':'),
mask(cast(12345 as smallint), -1, -1, -1, -1, '5'),
mask(cast(12345 as int), -1, -1, -1, -1, '5'),
mask(cast(12345 as bigint), -1, -1, -1, -1, '5'),
-   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 0, 0, 0)
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 0, 0, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, 0, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, -1, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, 0, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, -1, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, 0, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, -1, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, -1, -1)
 PREHOOK: type: QUERY
 

[1/2] hive git commit: HIVE-13196 : UDFLike: reduce Regex NFA sizes (Gopal V via Gunther Hagleitner)

[hashutosh]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 3fb299750 -> 74c81a892


HIVE-13196 : UDFLike: reduce Regex NFA sizes (Gopal V via Gunther Hagleitner)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/549fc26c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/549fc26c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/549fc26c

Branch: refs/heads/branch-2.1
Commit: 549fc26c933a36e4844f312d1c2cbfadfdd55537
Parents: 3fb2997
Author: Gopal V 
Authored: Tue May 31 11:06:44 2016 -0700
Committer: Ashutosh Chauhan 
Committed: Tue May 31 11:41:46 2016 -0700

--
 ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/549fc26c/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
index 7bcd36e..9ac9548 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
@@ -77,7 +77,7 @@ public class UDFLike extends UDF {
   if (n == '_') {
 sb.append(".");
   } else if (n == '%') {
-sb.append(".*");
+sb.append(".*?");
   } else {
 sb.append(Pattern.quote(Character.toString(n)));
   }

hive git commit: HIVE-13859: mask() UDF not retaining day and month field values (Madhan Neethiraj via Ashutosh Chauhan)

[hashutosh]

Repository: hive
Updated Branches:
  refs/heads/master ac9405004 -> a354507cc


HIVE-13859: mask() UDF not retaining day and month field values (Madhan 
Neethiraj via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a354507c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a354507c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a354507c

Branch: refs/heads/master
Commit: a354507ccf650a2b304a2ea76797e7846564e620
Parents: ac94050
Author: Madhan Neethiraj 
Authored: Sat May 28 03:16:38 2016 -0700
Committer: Ashutosh Chauhan 
Committed: Tue May 31 11:40:59 2016 -0700

--
 .../hive/ql/udf/generic/GenericUDFMask.java | 12 
 ql/src/test/queries/clientpositive/udf_mask.q   |  9 -
 .../test/results/clientpositive/udf_mask.q.out  | 20 +---
 3 files changed, 33 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/a354507c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java
index cd35142..b815aae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMask.java
@@ -96,12 +96,16 @@ class MaskTransformer extends AbstractTransformer {
   maskedNumber = MASKED_NUMBER;
 }
 
-if(maskedDayValue < 1 || maskedDayValue > 31) {
-  maskedDayValue = MASKED_DAY_COMPONENT_VAL;
+if(maskedDayValue != UNMASKED_VAL) {
+  if(maskedDayValue < 1 || maskedDayValue > 31) {
+maskedDayValue = MASKED_DAY_COMPONENT_VAL;
+  }
 }
 
-if(maskedMonthValue < 0 || maskedMonthValue > 11) {
-  maskedMonthValue = MASKED_MONTH_COMPONENT_VAL;
+if(maskedMonthValue != UNMASKED_VAL) {
+  if(maskedMonthValue < 0 || maskedMonthValue > 11) {
+maskedMonthValue = MASKED_MONTH_COMPONENT_VAL;
+  }
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a354507c/ql/src/test/queries/clientpositive/udf_mask.q
--
diff --git a/ql/src/test/queries/clientpositive/udf_mask.q 
b/ql/src/test/queries/clientpositive/udf_mask.q
index 82b8ee7..15f7d27 100644
--- a/ql/src/test/queries/clientpositive/udf_mask.q
+++ b/ql/src/test/queries/clientpositive/udf_mask.q
@@ -10,4 +10,11 @@ select mask('TestString-123', 'X', 'x', '0', ':'),
mask(cast(12345 as smallint), -1, -1, -1, -1, '5'),
mask(cast(12345 as int), -1, -1, -1, -1, '5'),
mask(cast(12345 as bigint), -1, -1, -1, -1, '5'),
-   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 0, 0, 0);
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 0, 0, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, 0, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, -1, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, 0, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, -1, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, 0, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, -1, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, -1, -1);

http://git-wip-us.apache.org/repos/asf/hive/blob/a354507c/ql/src/test/results/clientpositive/udf_mask.q.out
--
diff --git a/ql/src/test/results/clientpositive/udf_mask.q.out 
b/ql/src/test/results/clientpositive/udf_mask.q.out
index cbafaee..ac0e273 100644
--- a/ql/src/test/results/clientpositive/udf_mask.q.out
+++ b/ql/src/test/results/clientpositive/udf_mask.q.out
@@ -53,7 +53,14 @@ PREHOOK: query: select mask('TestString-123', 'X', 'x', '0', 
':'),
mask(cast(12345 as smallint), -1, -1, -1, -1, '5'),
mask(cast(12345 as int), -1, -1, -1, -1, '5'),
mask(cast(12345 as bigint), -1, -1, -1, -1, '5'),
-   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 0, 0, 0)
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 0, 0, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, 0, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, -1, 0),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, 0, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, 1, -1, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, 0, -1),
+   mask(cast('2016-04-20' as date), -1, -1, -1, -1, -1, -1, -1, 0),
+   

hive git commit: HIVE-13823: Remove unnecessary log line in common join operator (Gunther Hagleitner, reviewed by Prasanth Jayachandran)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 8aae75f65 -> 3fb299750


HIVE-13823: Remove unnecessary log line in common join operator (Gunther 
Hagleitner, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3fb29975
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3fb29975
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3fb29975

Branch: refs/heads/branch-2.1
Commit: 3fb2997501c594977bdc6ead33924bbf6a7ede1c
Parents: 8aae75f
Author: Gunther Hagleitner 
Authored: Tue May 31 19:17:02 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 19:17:02 2016 +0100

--
 .../org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/3fb29975/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
index 45bc0fd..002e49b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
@@ -446,7 +446,6 @@ public class CommonMergeJoinOperator extends 
AbstractMapJoinOperator

hive git commit: HIVE-13196 : UDFLike: reduce Regex NFA sizes (Gopal V via Gunther Hagleitner)

[hashutosh]

Repository: hive
Updated Branches:
  refs/heads/master e91f69e21 -> 982293550


HIVE-13196 : UDFLike: reduce Regex NFA sizes (Gopal V via Gunther Hagleitner)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/98229355
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/98229355
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/98229355

Branch: refs/heads/master
Commit: 98229355011cd1235c0c1e73b4f69643d7f502ec
Parents: e91f69e
Author: Gopal V 
Authored: Tue May 31 11:06:44 2016 -0700
Committer: Ashutosh Chauhan 
Committed: Tue May 31 11:06:44 2016 -0700

--
 ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/98229355/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
index 7bcd36e..9ac9548 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
@@ -77,7 +77,7 @@ public class UDFLike extends UDF {
   if (n == '_') {
 sb.append(".");
   } else if (n == '%') {
-sb.append(".*");
+sb.append(".*?");
   } else {
 sb.append(Pattern.quote(Character.toString(n)));
   }

hive git commit: HIVE-13844: Invalid index handler in org.apache.hadoop.hive.ql.index.HiveIndex class (Svetozar Ivanov, reviewed by Jesus Camacho Rodriguez)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 d0b9fe190 -> 8aae75f65


HIVE-13844: Invalid index handler in org.apache.hadoop.hive.ql.index.HiveIndex 
class (Svetozar Ivanov, reviewed by Jesus Camacho Rodriguez)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8aae75f6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8aae75f6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8aae75f6

Branch: refs/heads/branch-2.1
Commit: 8aae75f65e6ef286500bc989b953144dafbc30f4
Parents: d0b9fe1
Author: Svetozar Ivanov 
Authored: Tue May 31 16:35:09 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 16:35:09 2016 +0100

--
 .../apache/hadoop/hive/ql/index/HiveIndex.java  |  9 +++---
 .../hadoop/hive/ql/index/TestIndexType.java | 34 
 2 files changed, 39 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/8aae75f6/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java 
b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
index 36bc9cd..a1408e9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.ql.index;
 
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler;
+import org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,10 +32,9 @@ public class HiveIndex {
   public static String INDEX_TABLE_CREATETIME = 
"hive.index.basetbl.dfs.lastModifiedTime";
 
   public static enum IndexType {
-AGGREGATE_TABLE("aggregate", 
"org.apache.hadoop.hive.ql.AggregateIndexHandler"),
-COMPACT_SUMMARY_TABLE("compact", 
"org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler"),
-BITMAP_TABLE("bitmap",
-"org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler");
+AGGREGATE_TABLE("aggregate",  AggregateIndexHandler.class.getName()),
+COMPACT_SUMMARY_TABLE("compact", CompactIndexHandler.class.getName()),
+BITMAP_TABLE("bitmap", BitmapIndexHandler.class.getName());
 
 private IndexType(String indexType, String className) {
   indexTypeName = indexType;

http://git-wip-us.apache.org/repos/asf/hive/blob/8aae75f6/ql/src/test/org/apache/hadoop/hive/ql/index/TestIndexType.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/index/TestIndexType.java 
b/ql/src/test/org/apache/hadoop/hive/ql/index/TestIndexType.java
new file mode 100644
index 000..bc1f8d4
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/index/TestIndexType.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.index;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler;
+import org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler;
+import org.junit.Test;
+
+public class TestIndexType extends TestCase {
+
+@Test
+public void testIndexTypeHandlers(){
+assertEquals(HiveIndex.IndexType.AGGREGATE_TABLE.getHandlerClsName(), 
AggregateIndexHandler.class.getName());
+assertEquals(HiveIndex.IndexType.BITMAP_TABLE.getHandlerClsName(), 
BitmapIndexHandler.class.getName());
+
assertEquals(HiveIndex.IndexType.COMPACT_SUMMARY_TABLE.getHandlerClsName(), 
CompactIndexHandler.class.getName());
+}
+
+}

hive git commit: HIVE-13844: Invalid index handler in org.apache.hadoop.hive.ql.index.HiveIndex class (Svetozar Ivanov, reviewed by Jesus Camacho Rodriguez)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/master 411ab3feb -> e459a6728


HIVE-13844: Invalid index handler in org.apache.hadoop.hive.ql.index.HiveIndex 
class (Svetozar Ivanov, reviewed by Jesus Camacho Rodriguez)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e459a672
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e459a672
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e459a672

Branch: refs/heads/master
Commit: e459a67283900393a79e4f69853103cc4fd8a726
Parents: 411ab3f
Author: Svetozar Ivanov 
Authored: Tue May 31 16:31:19 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 16:33:09 2016 +0100

--
 .../apache/hadoop/hive/ql/index/HiveIndex.java  |  9 +++---
 .../hadoop/hive/ql/index/TestIndexType.java | 34 
 2 files changed, 39 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e459a672/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java 
b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
index 36bc9cd..a1408e9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndex.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.ql.index;
 
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler;
+import org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,10 +32,9 @@ public class HiveIndex {
   public static String INDEX_TABLE_CREATETIME = 
"hive.index.basetbl.dfs.lastModifiedTime";
 
   public static enum IndexType {
-AGGREGATE_TABLE("aggregate", 
"org.apache.hadoop.hive.ql.AggregateIndexHandler"),
-COMPACT_SUMMARY_TABLE("compact", 
"org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler"),
-BITMAP_TABLE("bitmap",
-"org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler");
+AGGREGATE_TABLE("aggregate",  AggregateIndexHandler.class.getName()),
+COMPACT_SUMMARY_TABLE("compact", CompactIndexHandler.class.getName()),
+BITMAP_TABLE("bitmap", BitmapIndexHandler.class.getName());
 
 private IndexType(String indexType, String className) {
   indexTypeName = indexType;

http://git-wip-us.apache.org/repos/asf/hive/blob/e459a672/ql/src/test/org/apache/hadoop/hive/ql/index/TestIndexType.java
--
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/index/TestIndexType.java 
b/ql/src/test/org/apache/hadoop/hive/ql/index/TestIndexType.java
new file mode 100644
index 000..bc1f8d4
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/index/TestIndexType.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.index;
+
+import junit.framework.TestCase;
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler;
+import org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler;
+import org.junit.Test;
+
+public class TestIndexType extends TestCase {
+
+@Test
+public void testIndexTypeHandlers(){
+assertEquals(HiveIndex.IndexType.AGGREGATE_TABLE.getHandlerClsName(), 
AggregateIndexHandler.class.getName());
+assertEquals(HiveIndex.IndexType.BITMAP_TABLE.getHandlerClsName(), 
BitmapIndexHandler.class.getName());
+
assertEquals(HiveIndex.IndexType.COMPACT_SUMMARY_TABLE.getHandlerClsName(), 
CompactIndexHandler.class.getName());
+}
+
+}

[2/2] hive git commit: HIVE-13693: Multi-insert query drops Filter before file output when there is a.val <> b.val (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

HIVE-13693: Multi-insert query drops Filter before file output when there is 
a.val <> b.val (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d0b9fe19
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d0b9fe19
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d0b9fe19

Branch: refs/heads/branch-2.1
Commit: d0b9fe1905d4caac3afe8e8ee3665161998327e3
Parents: c20a113
Author: Jesus Camacho Rodriguez 
Authored: Tue May 31 15:58:32 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 16:11:45 2016 +0100

--
 .../results/positive/hbase_ppd_key_range.q.out  | 140 ++---
 .../test/results/positive/hbase_pushdown.q.out  |  35 +-
 .../test/results/positive/ppd_key_ranges.q.out  |  70 +--
 .../hadoop/hive/ql/ppd/OpProcFactory.java   |  60 +-
 .../clientpositive/multi_insert_with_join2.q|  51 ++
 .../groupby_multi_single_reducer.q.out  | 112 ++--
 .../clientpositive/multi_insert_gby.q.out   |  50 +-
 .../multi_insert_with_join2.q.out   | 555 +++
 .../spark/groupby_multi_single_reducer.q.out| 112 ++--
 .../clientpositive/spark/multi_insert_gby.q.out |  50 +-
 .../spark/subquery_multiinsert.q.java1.7.out| 100 ++--
 .../spark/subquery_multiinsert.q.out|  70 +--
 .../subquery_multiinsert.q.java1.7.out  |  49 +-
 13 files changed, 1001 insertions(+), 453 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/d0b9fe19/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
--
diff --git a/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out 
b/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
index 0ef0efd..f92371d 100644
--- a/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
@@ -29,37 +29,22 @@ POSTHOOK: query: -- with full pushdown
 explain select * from hbase_pushdown where key>'90'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-0 depends on stages: Stage-1
+  Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-1
-Map Reduce
-  Map Operator Tree:
-  TableScan
-alias: hbase_pushdown
-filterExpr: (key > '90') (type: boolean)
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
-Filter Operator
-  predicate: (key > '90') (type: boolean)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
-  Select Operator
-expressions: key (type: string), value (type: string)
-outputColumnNames: _col0, _col1
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
-File Output Operator
-  compressed: false
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
-  table:
-  input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
-  output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
   Stage: Stage-0
 Fetch Operator
   limit: -1
   Processor Tree:
-ListSink
+TableScan
+  alias: hbase_pushdown
+  filterExpr: (key > '90') (type: boolean)
+  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Select Operator
+expressions: key (type: string), value (type: string)
+outputColumnNames: _col0, _col1
+Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+ListSink
 
 PREHOOK: query: select * from hbase_pushdown where key>'90'
 PREHOOK: type: QUERY
@@ -185,37 +170,22 @@ POSTHOOK: query: -- with cnostant expressinon
 explain select * from hbase_pushdown where key>=cast(40 + 50 as string)
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-0 depends on stages: Stage-1
+  Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-1
-Map Reduce
-  Map Operator Tree:
-  TableScan
-alias: hbase_pushdown
-filterExpr: (key >= '90') (type: boolean)
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
-Filter Operator
-  predicate: (key >= '90') (type: boolean)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
-  Select Operator

[1/2] hive git commit: HIVE-13693: Multi-insert query drops Filter before file output when there is a.val <> b.val (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/master aed350351 -> 411ab3feb


http://git-wip-us.apache.org/repos/asf/hive/blob/411ab3fe/ql/src/test/results/clientpositive/subquery_multiinsert.q.out
--
diff --git a/ql/src/test/results/clientpositive/subquery_multiinsert.q.out 
b/ql/src/test/results/clientpositive/subquery_multiinsert.q.out
index 62c5bf2..63f93fb 100644
--- a/ql/src/test/results/clientpositive/subquery_multiinsert.q.out
+++ b/ql/src/test/results/clientpositive/subquery_multiinsert.q.out
@@ -126,12 +126,15 @@ STAGE PLANS:
   sort order: 
   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: key (type: string), value (type: string)
-File Output Operator
-  compressed: false
-  table:
-  input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
-  output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-  serde: 
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+Filter Operator
+  predicate: (key is not null and value is not null) (type: 
boolean)
+  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE 
Column stats: NONE
+  File Output Operator
+compressed: false
+table:
+input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
+output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+serde: 
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
   TableScan
 Reduce Output Operator
   sort order: 
@@ -488,7 +491,7 @@ POSTHOOK: Input: default@src_5
 199val_199
 199val_199
 2  val_2
-Warning: Map Join MAPJOIN[55][bigTable=b] in task 'Stage-13:MAPRED' is a cross 
product
+Warning: Map Join MAPJOIN[56][bigTable=b] in task 'Stage-13:MAPRED' is a cross 
product
 Warning: Shuffle Join JOIN[31][tables = [b, sq_2_notin_nullcheck]] in Stage 
'Stage-2:MAPRED' is a cross product
 PREHOOK: query: explain
 from src b 
@@ -614,12 +617,15 @@ STAGE PLANS:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 serde: 
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-File Output Operator
-  compressed: false
-  table:
-  input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
-  output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-  serde: 
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+Filter Operator
+  predicate: (key is not null and value is not null) (type: 
boolean)
+  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE 
Column stats: NONE
+  File Output Operator
+compressed: false
+table:
+input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
+output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+serde: 
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
   Local Work:
 Map Reduce Local Work
 
@@ -773,12 +779,15 @@ STAGE PLANS:
   sort order: 
   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: key (type: string), value (type: string)
-File Output Operator
-  compressed: false
-  table:
-  input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
-  output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-  serde: 
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+Filter Operator
+  predicate: (key is not null and value is not null) (type: 
boolean)
+  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE 
Column stats: NONE
+  File Output Operator
+compressed: false
+table:
+input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
+output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+serde: 
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
   TableScan
 Reduce Output Operator
   sort order: 
@@ -799,7 +808,7 @@ STAGE PLANS:
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
 serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
-Warning: Map Join 

[2/2] hive git commit: HIVE-13693: Multi-insert query drops Filter before file output when there is a.val <> b.val (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

HIVE-13693: Multi-insert query drops Filter before file output when there is 
a.val <> b.val (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/411ab3fe
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/411ab3fe
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/411ab3fe

Branch: refs/heads/master
Commit: 411ab3febcb5fd06233cc82c6f95f8e680706356
Parents: aed3503
Author: Jesus Camacho Rodriguez 
Authored: Tue May 31 15:25:51 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 15:53:36 2016 +0100

--
 .../results/positive/hbase_ppd_key_range.q.out  | 140 ++---
 .../test/results/positive/hbase_pushdown.q.out  |  35 +-
 .../test/results/positive/ppd_key_ranges.q.out  |  70 +--
 .../hadoop/hive/ql/ppd/OpProcFactory.java   |  60 +-
 .../clientpositive/multi_insert_with_join2.q|  51 ++
 .../groupby_multi_single_reducer.q.out  | 112 ++--
 .../clientpositive/multi_insert_gby.q.out   |  50 +-
 .../multi_insert_with_join2.q.out   | 555 +++
 .../spark/groupby_multi_single_reducer.q.out| 112 ++--
 .../clientpositive/spark/multi_insert_gby.q.out |  50 +-
 .../spark/subquery_multiinsert.q.out|  70 +--
 .../clientpositive/subquery_multiinsert.q.out   |  49 +-
 12 files changed, 948 insertions(+), 406 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/411ab3fe/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
--
diff --git a/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out 
b/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
index 0ef0efd..f92371d 100644
--- a/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
@@ -29,37 +29,22 @@ POSTHOOK: query: -- with full pushdown
 explain select * from hbase_pushdown where key>'90'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-0 depends on stages: Stage-1
+  Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-1
-Map Reduce
-  Map Operator Tree:
-  TableScan
-alias: hbase_pushdown
-filterExpr: (key > '90') (type: boolean)
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
-Filter Operator
-  predicate: (key > '90') (type: boolean)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
-  Select Operator
-expressions: key (type: string), value (type: string)
-outputColumnNames: _col0, _col1
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
-File Output Operator
-  compressed: false
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL 
Column stats: NONE
-  table:
-  input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
-  output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
   Stage: Stage-0
 Fetch Operator
   limit: -1
   Processor Tree:
-ListSink
+TableScan
+  alias: hbase_pushdown
+  filterExpr: (key > '90') (type: boolean)
+  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+  Select Operator
+expressions: key (type: string), value (type: string)
+outputColumnNames: _col0, _col1
+Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
+ListSink
 
 PREHOOK: query: select * from hbase_pushdown where key>'90'
 PREHOOK: type: QUERY
@@ -185,37 +170,22 @@ POSTHOOK: query: -- with cnostant expressinon
 explain select * from hbase_pushdown where key>=cast(40 + 50 as string)
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-0 depends on stages: Stage-1
+  Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-1
-Map Reduce
-  Map Operator Tree:
-  TableScan
-alias: hbase_pushdown
-filterExpr: (key >= '90') (type: boolean)
-Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
-Filter Operator
-  predicate: (key >= '90') (type: boolean)
-  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column 
stats: NONE
-  Select Operator
-expressions: key (type: string), value (type: 

hive git commit: HIVE-13882: When hive.server2.async.exec.async.compile is turned on, from JDBC we will get "The query did not generate a result set" (Reviewed by Jimmy Xiang)

[aihuaxu]

Repository: hive
Updated Branches:
  refs/heads/master c6974c228 -> aed350351


HIVE-13882: When hive.server2.async.exec.async.compile is turned on, from JDBC 
we will get "The query did not generate a result set" (Reviewed by Jimmy Xiang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aed35035
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aed35035
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aed35035

Branch: refs/heads/master
Commit: aed350351b24125ff6bbf36373b5247070c48d5c
Parents: c6974c2
Author: Aihua Xu 
Authored: Tue Apr 26 14:31:55 2016 -0400
Committer: Aihua Xu 
Committed: Tue May 31 10:40:28 2016 -0400

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   2 +-
 .../apache/hive/jdbc/TestJdbcWithMiniHS2.java   |  69 +++-
 .../org/apache/hive/jdbc/HiveStatement.java |  12 ++-
 service-rpc/if/TCLIService.thrift   |   4 +
 .../gen/thrift/gen-cpp/TCLIService_types.cpp|  22 
 .../src/gen/thrift/gen-cpp/TCLIService_types.h  |  12 ++-
 .../rpc/thrift/TGetOperationStatusResp.java | 107 ++-
 service-rpc/src/gen/thrift/gen-php/Types.php|  23 
 .../src/gen/thrift/gen-py/TCLIService/ttypes.py |  15 ++-
 .../gen/thrift/gen-rb/t_c_l_i_service_types.rb  |   4 +-
 .../hive/service/cli/OperationStatus.java   |   8 +-
 .../hive/service/cli/operation/Operation.java   |   2 +-
 .../service/cli/thrift/ThriftCLIService.java|   1 +
 .../cli/thrift/ThriftCLIServiceClient.java  |   2 +-
 14 files changed, 264 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/aed35035/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 6a404bd..cdff4db 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2082,7 +2082,7 @@ public class HiveConf extends Configuration {
 HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", "",
 "Bind host on which to run the HiveServer2 Thrift service."),
 HIVE_SERVER2_PARALLEL_COMPILATION("hive.driver.parallel.compilation", 
false, "Whether to\n" +
-"enable parallel compilation between sessions on HiveServer2. The 
default is false."),
+"enable parallel compilation of the queries between sessions and 
within the same session on HiveServer2. The default is false."),
 HIVE_SERVER2_COMPILE_LOCK_TIMEOUT("hive.server2.compile.lock.timeout", 
"0s",
 new TimeValidator(TimeUnit.SECONDS),
 "Number of seconds a request will wait to acquire the compile lock 
before giving up. " +

http://git-wip-us.apache.org/repos/asf/hive/blob/aed35035/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java 
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
index a01daa4..0c313a2 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
@@ -50,6 +50,7 @@ import java.util.concurrent.TimeoutException;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -138,6 +139,66 @@ public class TestJdbcWithMiniHS2 {
   }
 
   @Test
+  public void testParallelCompilation() throws Exception {
+final String tableName = "testParallelCompilation";
+hs2Conn = getConnection();
+Statement stmt = hs2Conn.createStatement();
+
+// create table
+stmt.execute("DROP TABLE IF EXISTS " + tableName);
+stmt.execute("CREATE TABLE " + tableName
++ " (under_col INT COMMENT 'the under column', value STRING) COMMENT ' 
test table'");
+
+// load data
+stmt.execute("load data local inpath '"
++ kvDataFilePath.toString() + "' into table " + tableName);
+
+ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
+assertTrue(res.next());
+res.close();
+
+stmt.execute("SET hive.driver.parallel.compilation=true");
+stmt.execute("SET hive.server2.async.exec.async.compile=true");
+
+stmt.close();
+
+startConcurrencyTest(hs2Conn, tableName, 10);
+Connection conn2 = getConnection();
+startConcurrencyTest(conn2, tableName, 10);
+conn2.close();
+  }
+
+  @Test
+  public void 

hive git commit: HIVE-13831: Error pushing predicates to HBase storage handler (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 bb5fb1039 -> c20a113e4


HIVE-13831: Error pushing predicates to HBase storage handler (Jesus Camacho 
Rodriguez, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c20a113e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c20a113e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c20a113e

Branch: refs/heads/branch-2.1
Commit: c20a113e494387f62867906caf101bc9032004e7
Parents: bb5fb10
Author: Jesus Camacho Rodriguez 
Authored: Tue May 31 15:19:03 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 15:19:03 2016 +0100

--
 .../predicate/AccumuloPredicateHandler.java |   4 +-
 .../hive/accumulo/predicate/PushdownTuple.java  |   2 +-
 .../hive/hbase/AbstractHBaseKeyFactory.java |   6 +-
 .../hadoop/hive/hbase/HBaseStorageHandler.java  | 117 +--
 .../results/positive/external_table_ppd.q.out   |   1 -
 .../test/results/positive/hbase_pushdown.q.out  |   2 +-
 .../test/results/positive/hbase_timestamp.q.out |  12 +-
 .../test/results/positive/ppd_key_ranges.q.out  |   2 +-
 .../hive/ql/index/IndexPredicateAnalyzer.java   |  36 +-
 .../hive/ql/index/IndexSearchCondition.java |  33 --
 .../metadata/HiveStoragePredicateHandler.java   |   4 +-
 11 files changed, 178 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/c20a113e/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
--
diff --git 
a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
 
b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
index d5cc9a5..a7ec7c5 100644
--- 
a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
+++ 
b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
@@ -149,7 +149,7 @@ public class AccumuloPredicateHandler {
   return clz.newInstance();
 } catch (ClassCastException e) {
   throw new SerDeException("Column type mismatch in WHERE clause "
-  + sc.getComparisonExpr().getExprString() + " found type "
+  + sc.getIndexExpr().getExprString() + " found type "
   + sc.getConstantDesc().getTypeString() + " instead of "
   + sc.getColumnDesc().getTypeString());
 } catch (IllegalAccessException e) {
@@ -181,7 +181,7 @@ public class AccumuloPredicateHandler {
   return clz.newInstance();
 } catch (ClassCastException e) {
   throw new SerDeException("Column type mismatch in WHERE clause "
-  + sc.getComparisonExpr().getExprString() + " found type "
+  + sc.getIndexExpr().getExprString() + " found type "
   + sc.getConstantDesc().getTypeString() + " instead of "
   + sc.getColumnDesc().getTypeString());
 } catch (IllegalAccessException e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/c20a113e/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
--
diff --git 
a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
 
b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
index f326d52..085146d 100644
--- 
a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
+++ 
b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
@@ -60,7 +60,7 @@ public class PushdownTuple {
 } catch (ClassCastException cce) {
   log.info(StringUtils.stringifyException(cce));
   throw new SerDeException(" Column type mismatch in where clause "
-  + sc.getComparisonExpr().getExprString() + " found type "
+  + sc.getIndexExpr().getExprString() + " found type "
   + sc.getConstantDesc().getTypeString() + " instead of "
   + sc.getColumnDesc().getTypeString());
 } catch (HiveException e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/c20a113e/hbase-handler/src/java/org/apache/hadoop/hive/hbase/AbstractHBaseKeyFactory.java
--
diff --git 
a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/AbstractHBaseKeyFactory.java
 
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/AbstractHBaseKeyFactory.java
index 18fb5ea..4cc9619 100644
--- 
a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/AbstractHBaseKeyFactory.java
+++ 

hive git commit: HIVE-13831: Error pushing predicates to HBase storage handler (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/master 7f6dd6f9d -> c6974c228


HIVE-13831: Error pushing predicates to HBase storage handler (Jesus Camacho 
Rodriguez, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c6974c22
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c6974c22
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c6974c22

Branch: refs/heads/master
Commit: c6974c2288ee6638c23ea7df3c6adc1c1eb9fe70
Parents: 7f6dd6f
Author: Jesus Camacho Rodriguez 
Authored: Tue May 31 15:12:52 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 15:12:52 2016 +0100

--
 .../predicate/AccumuloPredicateHandler.java |   4 +-
 .../hive/accumulo/predicate/PushdownTuple.java  |   2 +-
 .../hive/hbase/AbstractHBaseKeyFactory.java |   6 +-
 .../hadoop/hive/hbase/HBaseStorageHandler.java  | 117 +--
 .../results/positive/external_table_ppd.q.out   |   1 -
 .../test/results/positive/hbase_pushdown.q.out  |   2 +-
 .../test/results/positive/hbase_timestamp.q.out |  12 +-
 .../test/results/positive/ppd_key_ranges.q.out  |   2 +-
 .../hive/ql/index/IndexPredicateAnalyzer.java   |  36 +-
 .../hive/ql/index/IndexSearchCondition.java |  33 --
 .../metadata/HiveStoragePredicateHandler.java   |   4 +-
 11 files changed, 178 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/c6974c22/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
--
diff --git 
a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
 
b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
index d5cc9a5..a7ec7c5 100644
--- 
a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
+++ 
b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/AccumuloPredicateHandler.java
@@ -149,7 +149,7 @@ public class AccumuloPredicateHandler {
   return clz.newInstance();
 } catch (ClassCastException e) {
   throw new SerDeException("Column type mismatch in WHERE clause "
-  + sc.getComparisonExpr().getExprString() + " found type "
+  + sc.getIndexExpr().getExprString() + " found type "
   + sc.getConstantDesc().getTypeString() + " instead of "
   + sc.getColumnDesc().getTypeString());
 } catch (IllegalAccessException e) {
@@ -181,7 +181,7 @@ public class AccumuloPredicateHandler {
   return clz.newInstance();
 } catch (ClassCastException e) {
   throw new SerDeException("Column type mismatch in WHERE clause "
-  + sc.getComparisonExpr().getExprString() + " found type "
+  + sc.getIndexExpr().getExprString() + " found type "
   + sc.getConstantDesc().getTypeString() + " instead of "
   + sc.getColumnDesc().getTypeString());
 } catch (IllegalAccessException e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/c6974c22/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
--
diff --git 
a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
 
b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
index f326d52..085146d 100644
--- 
a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
+++ 
b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/PushdownTuple.java
@@ -60,7 +60,7 @@ public class PushdownTuple {
 } catch (ClassCastException cce) {
   log.info(StringUtils.stringifyException(cce));
   throw new SerDeException(" Column type mismatch in where clause "
-  + sc.getComparisonExpr().getExprString() + " found type "
+  + sc.getIndexExpr().getExprString() + " found type "
   + sc.getConstantDesc().getTypeString() + " instead of "
   + sc.getColumnDesc().getTypeString());
 } catch (HiveException e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/c6974c22/hbase-handler/src/java/org/apache/hadoop/hive/hbase/AbstractHBaseKeyFactory.java
--
diff --git 
a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/AbstractHBaseKeyFactory.java
 
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/AbstractHBaseKeyFactory.java
index 18fb5ea..4cc9619 100644
--- 
a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/AbstractHBaseKeyFactory.java
+++ 
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/AbstractHBaseKeyFactory.java
@@ 

[4/4] hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product 
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bb5fb103
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bb5fb103
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bb5fb103

Branch: refs/heads/branch-2.1
Commit: bb5fb1039a104739efd5c2f2e57de8b8872c1f6f
Parents: 8e664dc
Author: Jesus Camacho Rodriguez 
Authored: Tue May 31 14:48:39 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 15:09:14 2016 +0100

--
 .../stats/annotation/StatsRulesProcFactory.java | 56 ++
 .../results/clientpositive/auto_join0.q.out | 10 ++--
 .../results/clientpositive/auto_join23.q.out| 10 ++--
 .../clientpositive/auto_sortmerge_join_12.q.out |  2 +-
 .../cbo_rp_cross_product_check_2.q.out  | 38 ++---
 .../results/clientpositive/cbo_rp_join1.q.out   | 12 ++--
 .../results/clientpositive/complex_alias.q.out  |  6 +-
 .../constantPropagateForSubQuery.q.out  |  6 +-
 .../clientpositive/constant_prop_1.q.out| 18 +++---
 .../results/clientpositive/constprog2.q.out |  6 +-
 .../clientpositive/constprog_partitioner.q.out  |  6 +-
 .../results/clientpositive/cross_join.q.out | 16 +++---
 .../clientpositive/cross_join_merge.q.out   | 12 ++--
 .../clientpositive/cross_product_check_1.q.out  | 30 +-
 .../clientpositive/cross_product_check_2.q.out  | 36 ++--
 .../test/results/clientpositive/cte_mat_1.q.out |  6 +-
 .../test/results/clientpositive/cte_mat_2.q.out |  6 +-
 .../test/results/clientpositive/input23.q.out   |  8 +--
 .../results/clientpositive/join0.q.java1.7.out  | 10 ++--
 ql/src/test/results/clientpositive/join23.q.out | 10 ++--
 ql/src/test/results/clientpositive/join42.q.out |  4 +-
 .../clientpositive/join_alt_syntax.q.out| 18 +++---
 .../clientpositive/join_cond_pushdown_1.q.out   | 16 +++---
 .../clientpositive/join_cond_pushdown_3.q.out   | 16 +++---
 .../join_cond_pushdown_unqual1.q.out| 20 +++
 .../join_cond_pushdown_unqual3.q.out| 20 +++
 .../results/clientpositive/llap/cte_mat_1.q.out |  6 +-
 .../results/clientpositive/llap/cte_mat_2.q.out |  6 +-
 .../llap/dynamic_partition_pruning.q.out| 10 ++--
 .../llap/dynamic_partition_pruning_2.q.out  |  4 +-
 .../llap/hybridgrace_hashjoin_1.q.out   | 12 ++--
 .../clientpositive/llap/tez_self_join.q.out |  6 +-
 .../vectorized_dynamic_partition_pruning.q.out  | 10 ++--
 .../test/results/clientpositive/no_hooks.q.out  | 10 ++--
 .../nonblock_op_deduplicate.q.out   |  8 +--
 .../clientpositive/optimize_nullscan.q.out  |  4 +-
 .../results/clientpositive/parallel_join0.q.out | 10 ++--
 .../results/clientpositive/perf/query28.q.out   |  6 +-
 .../results/clientpositive/perf/query88.q.out   |  4 +-
 .../results/clientpositive/perf/query90.q.out   |  8 +--
 .../results/clientpositive/pointlookup2.q.out   | 20 +++
 .../results/clientpositive/pointlookup3.q.out   | 10 ++--
 .../test/results/clientpositive/ppd_join5.q.out | 14 ++---
 .../clientpositive/ppd_outer_join5.q.out| 18 +++---
 .../results/clientpositive/ppd_udf_case.q.out   | 16 +++---
 .../results/clientpositive/smb_mapjoin_25.q.out | 40 ++---
 .../clientpositive/spark/auto_join0.q.out   |  6 +-
 .../clientpositive/spark/auto_join23.q.out  | 10 ++--
 .../spark/auto_sortmerge_join_12.q.out  |  2 +-
 .../clientpositive/spark/cross_join.q.out   | 16 +++---
 .../spark/cross_product_check_1.q.out   | 30 +-
 .../spark/cross_product_check_2.q.out   | 28 -
 .../results/clientpositive/spark/join0.q.out|  8 +--
 .../results/clientpositive/spark/join23.q.out   | 10 ++--
 .../clientpositive/spark/join_alt_syntax.q.out  | 18 +++---
 .../spark/join_cond_pushdown_1.q.out| 16 +++---
 .../spark/join_cond_pushdown_3.q.out| 16 +++---
 .../spark/join_cond_pushdown_unqual1.q.out  | 20 +++
 .../spark/join_cond_pushdown_unqual3.q.out  | 20 +++
 .../spark/optimize_nullscan.q.out   |  4 +-
 .../clientpositive/spark/parallel_join0.q.out   |  8 +--
 .../clientpositive/spark/ppd_join5.q.out| 14 ++---
 .../clientpositive/spark/ppd_outer_join5.q.out  | 18 +++---
 .../clientpositive/spark/smb_mapjoin_25.q.out   | 24 
 .../spark/subquery_multiinsert.q.out| 30 +-
 .../subquery_multiinsert.q.java1.7.out  | 32 +--
 .../results/clientpositive/subquery_notin.q.out | 60 ++--
 .../subquery_notin_having.q.java1.7.out | 36 ++--
 .../subquery_unqual_corr_expr.q.out | 12 ++--
 .../subquery_unqualcolumnrefs.q.out | 12 ++--
 

[2/4] hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

http://git-wip-us.apache.org/repos/asf/hive/blob/bb5fb103/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out 
b/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
index 9a965dd..2e4cee1 100644
--- a/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
@@ -83,10 +83,10 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2, _col3
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 5000 Data size: 110120 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 5000 Data size: 110120 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -187,10 +187,10 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 5500 Data size: 120932 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 5500 Data size: 120932 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -282,10 +282,10 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2500 Data size: 54560 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2500 Data size: 54560 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -392,10 +392,10 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 25000 Data size: 675600 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 25000 Data size: 675600 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -409,27 +409,27 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0
-Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 100 Data size: 1540 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   keys: _col0 (type: string)
   mode: hash
   outputColumnNames: _col0
-  Statistics: Num rows: 11 Data size: 114 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 100 Data size: 1540 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: string)
 sort order: +
 Map-reduce partition columns: _col0 (type: string)
-Statistics: Num rows: 11 Data size: 114 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 100 Data size: 1540 Basic stats: 
COMPLETE Column stats: NONE

[3/4] hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

http://git-wip-us.apache.org/repos/asf/hive/blob/bb5fb103/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
--
diff --git a/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out 
b/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
index 771745f..66a300e 100644
--- a/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
+++ b/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
@@ -227,10 +227,10 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, 
_col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, 
_col17
-  Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 676 Data size: 164320 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: ((_col9 + _col0) = _col0) (type: boolean)
-Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 338 Data size: 82160 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
   table:
@@ -246,7 +246,7 @@ STAGE PLANS:
   key expressions: _col10 (type: string)
   sort order: +
   Map-reduce partition columns: _col10 (type: string)
-  Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 338 Data size: 82160 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: _col0 (type: int), _col1 (type: string), 
_col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: 
int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 
(type: int), _col11 (type: string), _col12 (type: string), _col13 (type: 
string), _col14 (type: int), _col15 (type: string), _col16 (type: double), 
_col17 (type: string)
   TableScan
 alias: p1
@@ -272,10 +272,10 @@ STAGE PLANS:
 0 _col10 (type: string)
 1 _col1 (type: string)
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, 
_col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, 
_col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
-  Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 371 Data size: 90376 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 371 Data size: 90376 Basic stats: COMPLETE 
Column stats: NONE
 table:
 input format: org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -380,14 +380,14 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, 
_col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, 
_col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
-  Statistics: Num rows: 30 Data size: 3807 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 728 Data size: 178830 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: _col18 (type: int), _col19 (type: string), _col20 
(type: string), _col21 (type: string), _col22 (type: string), _col23 (type: 
int), _col24 (type: string), _col25 (type: double), _col26 (type: string), 1 
(type: int), _col10 (type: string), _col11 (type: string), _col12 (type: 
string), _col13 (type: string), _col14 (type: int), _col15 (type: string), 
_col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: 
string), _col2 (type: string), _col3 (type: string), _col4 (type: string), 
_col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: 
string)
 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, 
_col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, 
_col26
-Statistics: Num rows: 30 Data size: 3807 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 728 Data size: 178830 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 30 Data size: 3807 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 728 Data size: 178830 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input 

[1/4] hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/branch-2.1 8e664dca0 -> bb5fb1039


http://git-wip-us.apache.org/repos/asf/hive/blob/bb5fb103/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
--
diff --git 
a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out 
b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
index 793b8be..b931291 100644
--- a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
@@ -87,7 +87,7 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 250 Data size: 4906 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
 table:
@@ -103,7 +103,7 @@ STAGE PLANS:
   key expressions: _col0 (type: string)
   sort order: +
   Map-reduce partition columns: _col0 (type: string)
-  Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 250 Data size: 4906 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: _col1 (type: bigint)
   TableScan
 alias: src
@@ -128,17 +128,17 @@ STAGE PLANS:
 0 _col0 (type: string)
 1 _col0 (type: string)
   outputColumnNames: _col0, _col1, _col3
-  Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 275 Data size: 5396 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: _col3 is null (type: boolean)
-Statistics: Num rows: 151 Data size: 1606 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 137 Data size: 2688 Basic stats: COMPLETE 
Column stats: NONE
 Select Operator
   expressions: _col0 (type: string), _col1 (type: bigint)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 151 Data size: 1606 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 137 Data size: 2688 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 151 Data size: 1606 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 137 Data size: 2688 Basic stats: 
COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -277,7 +277,7 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 13 Data size: 1898 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
 table:
@@ -293,7 +293,7 @@ STAGE PLANS:
   key expressions: _col0 (type: string), _col1 (type: double)
   sort order: ++
   Map-reduce partition columns: _col0 (type: string), _col1 (type: 
double)
-  Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 13 Data size: 1898 Basic stats: COMPLETE 
Column stats: NONE
   TableScan
 Reduce Output Operator
   key expressions: _col0 (type: string), _col1 (type: double)
@@ -308,17 +308,17 @@ STAGE PLANS:
 0 _col0 (type: string), _col1 (type: double)
 1 _col0 (type: string), _col1 (type: double)
   outputColumnNames: _col0, _col1, _col3
-  Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 14 Data size: 2087 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: _col3 is null (type: boolean)
-Statistics: Num rows: 7 Data size: 888 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 7 Data size: 1043 Basic stats: COMPLETE 
Column stats: NONE
 Select Operator
   expressions: _col0 (type: string), _col1 (type: double)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 7 Data size: 888 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 7 Data size: 1043 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 7 Data size: 888 Basic 

[1/4] hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

Repository: hive
Updated Branches:
  refs/heads/master 4d988b195 -> 7f6dd6f9d


http://git-wip-us.apache.org/repos/asf/hive/blob/7f6dd6f9/ql/src/test/results/clientpositive/subquery_notin.q.out
--
diff --git a/ql/src/test/results/clientpositive/subquery_notin.q.out 
b/ql/src/test/results/clientpositive/subquery_notin.q.out
index fed1f89..6873aca 100644
--- a/ql/src/test/results/clientpositive/subquery_notin.q.out
+++ b/ql/src/test/results/clientpositive/subquery_notin.q.out
@@ -88,7 +88,7 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 500 Data size: 9812 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
 table:
@@ -104,7 +104,7 @@ STAGE PLANS:
   key expressions: _col0 (type: string)
   sort order: +
   Map-reduce partition columns: _col0 (type: string)
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 500 Data size: 9812 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: _col1 (type: string)
   TableScan
 alias: src
@@ -129,17 +129,17 @@ STAGE PLANS:
 0 _col0 (type: string)
 1 _col0 (type: string)
   outputColumnNames: _col0, _col1, _col3
-  Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 550 Data size: 10793 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: _col3 is null (type: boolean)
-Statistics: Num rows: 302 Data size: 3208 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 275 Data size: 5396 Basic stats: COMPLETE 
Column stats: NONE
 Select Operator
   expressions: _col0 (type: string), _col1 (type: string)
   outputColumnNames: _col0, _col1
-  Statistics: Num rows: 302 Data size: 3208 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 275 Data size: 5396 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 302 Data size: 3208 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 275 Data size: 5396 Basic stats: 
COMPLETE Column stats: NONE
 table:
 input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -423,7 +423,7 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col0, _col1, _col2
-  Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 26 Data size: 3381 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
 table:
@@ -439,7 +439,7 @@ STAGE PLANS:
   key expressions: _col0 (type: string), _col1 (type: string)
   sort order: ++
   Map-reduce partition columns: _col0 (type: string), _col1 (type: 
string)
-  Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 26 Data size: 3381 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: _col2 (type: int)
   TableScan
 Reduce Output Operator
@@ -455,17 +455,17 @@ STAGE PLANS:
 0 _col0 (type: string), _col1 (type: string)
 1 _col0 (type: string), _col1 (type: string)
   outputColumnNames: _col0, _col1, _col2, _col4
-  Statistics: Num rows: 30 Data size: 3807 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 28 Data size: 3719 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: _col4 is null (type: boolean)
-Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 14 Data size: 1859 Basic stats: COMPLETE 
Column stats: NONE
 Select Operator
   expressions: _col1 (type: string), _col0 (type: string), _col2 
(type: int)
   outputColumnNames: _col0, _col1, _col2
-  Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 14 Data size: 1859 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 15 Data size: 1903 Basic stats: COMPLETE 
Column stats: NONE
+

[4/4] hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product 
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7f6dd6f9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7f6dd6f9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7f6dd6f9

Branch: refs/heads/master
Commit: 7f6dd6f9df171d9a0543d00f4eb8a6830b459ac3
Parents: 4d988b1
Author: Jesus Camacho Rodriguez 
Authored: Mon May 30 15:21:37 2016 +0100
Committer: Jesus Camacho Rodriguez 
Committed: Tue May 31 14:44:48 2016 +0100

--
 .../stats/annotation/StatsRulesProcFactory.java |  56 --
 .../results/clientpositive/auto_join0.q.out |  10 +-
 .../results/clientpositive/auto_join23.q.out|  10 +-
 .../clientpositive/auto_sortmerge_join_12.q.out |   2 +-
 .../cbo_rp_cross_product_check_2.q.out  |  38 ++--
 .../results/clientpositive/cbo_rp_join1.q.out   |  12 +-
 .../results/clientpositive/complex_alias.q.out  |   6 +-
 .../constantPropagateForSubQuery.q.out  |   6 +-
 .../clientpositive/constant_prop_1.q.out|  18 +-
 .../results/clientpositive/constprog2.q.out |   6 +-
 .../clientpositive/constprog_partitioner.q.out  |   6 +-
 .../results/clientpositive/cross_join.q.out |  16 +-
 .../clientpositive/cross_join_merge.q.out   |  12 +-
 .../clientpositive/cross_product_check_1.q.out  |  30 ++--
 .../clientpositive/cross_product_check_2.q.out  |  36 ++--
 .../test/results/clientpositive/cte_mat_1.q.out |   6 +-
 .../test/results/clientpositive/cte_mat_2.q.out |   6 +-
 .../test/results/clientpositive/input23.q.out   |   8 +-
 ql/src/test/results/clientpositive/join0.q.out  |  10 +-
 ql/src/test/results/clientpositive/join23.q.out |  10 +-
 ql/src/test/results/clientpositive/join42.q.out |   4 +-
 .../clientpositive/join_alt_syntax.q.out|  18 +-
 .../clientpositive/join_cond_pushdown_1.q.out   |  16 +-
 .../clientpositive/join_cond_pushdown_3.q.out   |  16 +-
 .../join_cond_pushdown_unqual1.q.out|  20 +--
 .../join_cond_pushdown_unqual3.q.out|  20 +--
 .../results/clientpositive/llap/cte_mat_1.q.out |   6 +-
 .../results/clientpositive/llap/cte_mat_2.q.out |   6 +-
 .../llap/dynamic_partition_pruning.q.out|  10 +-
 .../llap/dynamic_partition_pruning_2.q.out  |   4 +-
 .../llap/hybridgrace_hashjoin_1.q.out   |  12 +-
 .../clientpositive/llap/tez_self_join.q.out |   6 +-
 .../vectorized_dynamic_partition_pruning.q.out  |  10 +-
 .../test/results/clientpositive/no_hooks.q.out  |  10 +-
 .../nonblock_op_deduplicate.q.out   |   8 +-
 .../clientpositive/optimize_nullscan.q.out  |   4 +-
 .../results/clientpositive/parallel_join0.q.out |  10 +-
 .../results/clientpositive/perf/query28.q.out   |   6 +-
 .../results/clientpositive/perf/query88.q.out   |   4 +-
 .../results/clientpositive/perf/query90.q.out   |   8 +-
 .../results/clientpositive/pointlookup2.q.out   |  20 +--
 .../results/clientpositive/pointlookup3.q.out   |  10 +-
 .../test/results/clientpositive/ppd_join5.q.out |  14 +-
 .../clientpositive/ppd_outer_join5.q.out|  18 +-
 .../results/clientpositive/ppd_udf_case.q.out   |  16 +-
 .../results/clientpositive/smb_mapjoin_25.q.out |  40 ++---
 .../clientpositive/spark/auto_join0.q.out   |   6 +-
 .../clientpositive/spark/auto_join23.q.out  |  10 +-
 .../spark/auto_sortmerge_join_12.q.out  |   2 +-
 .../clientpositive/spark/cross_join.q.out   |  16 +-
 .../spark/cross_product_check_1.q.out   |  30 ++--
 .../spark/cross_product_check_2.q.out   |  28 +--
 .../results/clientpositive/spark/join0.q.out|   8 +-
 .../results/clientpositive/spark/join23.q.out   |  10 +-
 .../clientpositive/spark/join_alt_syntax.q.out  |  18 +-
 .../spark/join_cond_pushdown_1.q.out|  16 +-
 .../spark/join_cond_pushdown_3.q.out|  16 +-
 .../spark/join_cond_pushdown_unqual1.q.out  |  20 +--
 .../spark/join_cond_pushdown_unqual3.q.out  |  20 +--
 .../spark/optimize_nullscan.q.out   |   4 +-
 .../clientpositive/spark/parallel_join0.q.out   |   8 +-
 .../clientpositive/spark/ppd_join5.q.out|  14 +-
 .../clientpositive/spark/ppd_outer_join5.q.out  |  18 +-
 .../clientpositive/spark/smb_mapjoin_25.q.out   |  24 +--
 .../spark/subquery_multiinsert.q.out|  30 ++--
 .../clientpositive/subquery_multiinsert.q.out   | 176 +--
 .../results/clientpositive/subquery_notin.q.out |  60 +++
 .../clientpositive/subquery_notin_having.q.out  |  36 ++--
 .../subquery_unqual_corr_expr.q.out |  12 +-
 .../subquery_unqualcolumnrefs.q.out |  12 +-
 .../results/clientpositive/subquery_views.q.out |  30 ++--
 .../results/clientpositive/tez/auto_join0.q.out |   6 +-
 

[3/4] hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

http://git-wip-us.apache.org/repos/asf/hive/blob/7f6dd6f9/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
--
diff --git a/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out 
b/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
index 771745f..66a300e 100644
--- a/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
+++ b/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
@@ -227,10 +227,10 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, 
_col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, 
_col17
-  Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 676 Data size: 164320 Basic stats: COMPLETE 
Column stats: NONE
   Filter Operator
 predicate: ((_col9 + _col0) = _col0) (type: boolean)
-Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 338 Data size: 82160 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
   table:
@@ -246,7 +246,7 @@ STAGE PLANS:
   key expressions: _col10 (type: string)
   sort order: +
   Map-reduce partition columns: _col10 (type: string)
-  Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 338 Data size: 82160 Basic stats: COMPLETE 
Column stats: NONE
   value expressions: _col0 (type: int), _col1 (type: string), 
_col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: 
int), _col6 (type: string), _col7 (type: double), _col8 (type: string), _col9 
(type: int), _col11 (type: string), _col12 (type: string), _col13 (type: 
string), _col14 (type: int), _col15 (type: string), _col16 (type: double), 
_col17 (type: string)
   TableScan
 alias: p1
@@ -272,10 +272,10 @@ STAGE PLANS:
 0 _col10 (type: string)
 1 _col1 (type: string)
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, 
_col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, 
_col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
-  Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 371 Data size: 90376 Basic stats: COMPLETE 
Column stats: NONE
   File Output Operator
 compressed: false
-Statistics: Num rows: 28 Data size: 3461 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 371 Data size: 90376 Basic stats: COMPLETE 
Column stats: NONE
 table:
 input format: org.apache.hadoop.mapred.SequenceFileInputFormat
 output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -380,14 +380,14 @@ STAGE PLANS:
 0 
 1 
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, 
_col7, _col8, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, 
_col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26
-  Statistics: Num rows: 30 Data size: 3807 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 728 Data size: 178830 Basic stats: COMPLETE 
Column stats: NONE
   Select Operator
 expressions: _col18 (type: int), _col19 (type: string), _col20 
(type: string), _col21 (type: string), _col22 (type: string), _col23 (type: 
int), _col24 (type: string), _col25 (type: double), _col26 (type: string), 1 
(type: int), _col10 (type: string), _col11 (type: string), _col12 (type: 
string), _col13 (type: string), _col14 (type: int), _col15 (type: string), 
_col16 (type: double), _col17 (type: string), _col0 (type: int), _col1 (type: 
string), _col2 (type: string), _col3 (type: string), _col4 (type: string), 
_col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: 
string)
 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, 
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, 
_col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, 
_col26
-Statistics: Num rows: 30 Data size: 3807 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 728 Data size: 178830 Basic stats: COMPLETE 
Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 30 Data size: 3807 Basic stats: COMPLETE 
Column stats: NONE
+  Statistics: Num rows: 728 Data size: 178830 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input 

[2/4] hive git commit: HIVE-13863: Improve AnnotateWithStatistics with support for cartesian product (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)

[jcamacho]

http://git-wip-us.apache.org/repos/asf/hive/blob/7f6dd6f9/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out 
b/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
index 9a965dd..2e4cee1 100644
--- a/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/cross_product_check_1.q.out
@@ -83,10 +83,10 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2, _col3
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 5000 Data size: 110120 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 5000 Data size: 110120 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -187,10 +187,10 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 5500 Data size: 120932 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 5500 Data size: 120932 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -282,10 +282,10 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 2500 Data size: 54560 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 2500 Data size: 54560 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -392,10 +392,10 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0, _col1, _col2
-Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 25000 Data size: 675600 Basic stats: 
COMPLETE Column stats: NONE
 File Output Operator
   compressed: false
-  Statistics: Num rows: 550 Data size: 5843 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 25000 Data size: 675600 Basic stats: 
COMPLETE Column stats: NONE
   table:
   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -409,27 +409,27 @@ STAGE PLANS:
   0 
   1 
 outputColumnNames: _col0
-Statistics: Num rows: 11 Data size: 114 Basic stats: COMPLETE 
Column stats: NONE
+Statistics: Num rows: 100 Data size: 1540 Basic stats: 
COMPLETE Column stats: NONE
 Group By Operator
   keys: _col0 (type: string)
   mode: hash
   outputColumnNames: _col0
-  Statistics: Num rows: 11 Data size: 114 Basic stats: 
COMPLETE Column stats: NONE
+  Statistics: Num rows: 100 Data size: 1540 Basic stats: 
COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: string)
 sort order: +
 Map-reduce partition columns: _col0 (type: string)
-Statistics: Num rows: 11 Data size: 114 Basic stats: 
COMPLETE Column stats: NONE
+Statistics: Num rows: 100 Data size: 1540 Basic stats: 
COMPLETE Column stats: NONE