date:20160407

hive git commit: HIVE-13333: StatsOptimizer throws ClassCastException (Pengcheng Xiong, reviewed by Ashutosh Chauhan)

2016-04-07 Thread pxiong

Repository: hive
Updated Branches:
  refs/heads/master 12cb6a98e -> fee6669c1


HIVE-1: StatsOptimizer throws ClassCastException (Pengcheng Xiong, reviewed 
by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fee6669c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fee6669c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fee6669c

Branch: refs/heads/master
Commit: fee6669c183b5635509336fd6d7f625a97bb2921
Parents: 12cb6a9
Author: Pengcheng Xiong 
Authored: Thu Apr 7 17:09:27 2016 -0700
Committer: Pengcheng Xiong 
Committed: Thu Apr 7 17:09:27 2016 -0700

--
 .../hive/ql/optimizer/StatsOptimizer.java   |  37 +-
 .../clientpositive/cbo_rp_udf_udaf_stats_opt.q  |  22 
 .../cbo_rp_udf_udaf_stats_opt.q.out | 126 +++
 3 files changed, 183 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/fee6669c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
index bc17fec..0cfd529 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
@@ -19,11 +19,14 @@ package org.apache.hadoop.hive.ql.optimizer;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.Stack;
 
@@ -665,7 +668,37 @@ public class StatsOptimizer extends Transform {
 
ois.add(TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(colInfo.getType()));
   }
 } else {
-  int aggrPos = 0;
+  // in return path, we may have aggr($f0), aggr($f1) in GBY
+  // and then select aggr($f1), aggr($f0) in SEL.
+  // Thus we need to use colExp to find out which position is
+  // corresponding to which position.
+  Map nameToIndex = new HashMap<>();
+  for (int index = 0; index < 
cgbyOp.getConf().getOutputColumnNames().size(); index++) {
+
nameToIndex.put(cgbyOp.getConf().getOutputColumnNames().get(index), index);
+  }
+  List outputColumnNames = 
cselOp.getConf().getOutputColumnNames();
+  Map cselOpTocgbyOp = new HashMap<>();
+  for (int index = 0; index < outputColumnNames.size(); index++) {
+if (!posToConstant.containsKey(index)) {
+  String outputColumnName = outputColumnNames.get(index);
+  ExprNodeColumnDesc exprColumnNodeDesc = (ExprNodeColumnDesc) 
cselOp
+  .getColumnExprMap().get(outputColumnName);
+  cselOpTocgbyOp.put(index, 
nameToIndex.get(exprColumnNodeDesc.getColumn()));
+}
+  }
+  // cselOpTocgbyOp may be 0 to 1, where the 0th position of cgbyOp is 
'1' and 1st position of cgbyOp is count('1')
+  // Thus, we need to adjust it to the correct position.
+  List> list = new 
ArrayList<>(cselOpTocgbyOp.entrySet());
+  Collections.sort(list, new Comparator>() {
+public int compare(Entry o1, Entry o2) {
+  return (o1.getValue()).compareTo(o2.getValue());
+}
+  });
+  cselOpTocgbyOp.clear();
+  // adjust cselOpTocgbyOp
+  for (int index = 0; index < list.size(); index++) {
+cselOpTocgbyOp.put(list.get(index).getKey(), index);
+  }
   List oneRowWithConstant = new ArrayList<>();
   for (int pos = 0; pos < cselOp.getSchema().getSignature().size(); 
pos++) {
 if (posToConstant.containsKey(pos)) {
@@ -673,7 +706,7 @@ public class StatsOptimizer extends Transform {
   oneRowWithConstant.add(posToConstant.get(pos));
 } else {
   // This position is an aggregation.
-  oneRowWithConstant.add(oneRow.get(aggrPos++));
+  oneRowWithConstant.add(oneRow.get(cselOpTocgbyOp.get(pos)));
 }
 ColumnInfo colInfo = cselOp.getSchema().getSignature().get(pos);
 colNames.add(colInfo.getInternalName());

http://git-wip-us.apache.org/repos/asf/hive/blob/fee6669c/ql/src/test/queries/clientpositive/cbo_rp_udf_udaf_stats_opt.q

hive git commit: HIVE-13417: Some vector operators return 'OP' as name (Gunther Hagleitner, reviewed by Vikram Dixit K)

2016-04-07 Thread gunther

Repository: hive
Updated Branches:
  refs/heads/master 7e0b08c1d -> 12cb6a98e


HIVE-13417: Some vector operators return 'OP' as name (Gunther Hagleitner, 
reviewed by Vikram Dixit K)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/12cb6a98
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/12cb6a98
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/12cb6a98

Branch: refs/heads/master
Commit: 12cb6a98e20fff9a1e0d8c615c76af2dc85ebce4
Parents: 7e0b08c
Author: Gunther Hagleitner 
Authored: Mon Apr 4 10:51:09 2016 -0700
Committer: Gunther Hagleitner 
Committed: Thu Apr 7 14:44:04 2016 -0700

--
 .../hive/ql/exec/AbstractFileMergeOperator.java |  9 ++
 .../hive/ql/exec/AppMasterEventOperator.java|  4 +-
 .../hadoop/hive/ql/exec/CollectOperator.java| 11 ++-
 .../hadoop/hive/ql/exec/CommonJoinOperator.java |  8 +-
 .../hadoop/hive/ql/exec/DemuxOperator.java  |  8 +-
 .../hadoop/hive/ql/exec/DummyStoreOperator.java | 11 ++-
 .../hadoop/hive/ql/exec/FilterOperator.java |  4 +-
 .../hadoop/hive/ql/exec/ForwardOperator.java|  4 +-
 .../hadoop/hive/ql/exec/GroupByOperator.java|  4 +-
 .../hive/ql/exec/HashTableDummyOperator.java|  4 +-
 .../ql/exec/LateralViewForwardOperator.java |  5 +-
 .../hive/ql/exec/LateralViewJoinOperator.java   |  4 +-
 .../hadoop/hive/ql/exec/LimitOperator.java  |  4 +-
 .../hadoop/hive/ql/exec/ListSinkOperator.java   | 11 ++-
 .../apache/hadoop/hive/ql/exec/MapOperator.java |  9 +-
 .../apache/hadoop/hive/ql/exec/MuxOperator.java |  8 +-
 .../apache/hadoop/hive/ql/exec/Operator.java| 22 +++--
 .../apache/hadoop/hive/ql/exec/PTFOperator.java |  4 +-
 .../hadoop/hive/ql/exec/ScriptOperator.java | 40 
 .../hadoop/hive/ql/exec/SelectOperator.java |  4 +-
 .../ql/exec/SparkHashTableSinkOperator.java | 10 +-
 .../hadoop/hive/ql/exec/TableScanOperator.java  |  2 +-
 .../hadoop/hive/ql/exec/TerminalOperator.java   | 10 ++
 .../hadoop/hive/ql/exec/UDTFOperator.java   |  8 +-
 .../hadoop/hive/ql/exec/UnionOperator.java  |  2 +-
 .../vector/VectorAppMasterEventOperator.java| 11 +--
 .../ql/exec/vector/VectorFileSinkOperator.java  |  8 +-
 .../ql/exec/vector/VectorFilterOperator.java|  8 +-
 .../ql/exec/vector/VectorGroupByOperator.java   | 25 +++--
 .../ql/exec/vector/VectorLimitOperator.java |  5 +-
 .../ql/exec/vector/VectorMapJoinOperator.java   | 11 ++-
 .../VectorMapJoinOuterFilteredOperator.java |  8 +-
 .../hive/ql/exec/vector/VectorMapOperator.java  |  5 +-
 .../exec/vector/VectorReduceSinkOperator.java   |  8 +-
 .../exec/vector/VectorSMBMapJoinOperator.java   | 13 +--
 .../ql/exec/vector/VectorSelectOperator.java| 21 +++--
 .../VectorSparkHashTableSinkOperator.java   |  6 +-
 ...VectorSparkPartitionPruningSinkOperator.java |  8 +-
 .../hive/ql/optimizer/physical/LlapDecider.java | 12 ++-
 .../SparkPartitionPruningSinkOperator.java  | 16 ++--
 .../hadoop/hive/ql/exec/TestOperatorNames.java  | 98 
 .../vector/util/FakeCaptureOutputOperator.java  | 10 +-
 .../util/FakeVectorDataSourceOperator.java  | 11 ++-
 .../clientpositive/explain_logical.q.out|  6 +-
 .../clientpositive/tez/explainuser_3.q.out  |  8 +-
 .../tez/vector_aggregate_without_gby.q.out  |  6 +-
 .../tez/vector_auto_smb_mapjoin_14.q.out| 38 
 .../tez/vector_groupby_mapjoin.q.out|  4 +-
 .../tez/vector_join_part_col_char.q.out | 10 +-
 .../tez/vectorized_parquet_types.q.out  |  2 +-
 .../tez/vectorized_timestamp.q.out  |  8 +-
 51 files changed, 360 insertions(+), 216 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
index 154a78b..dfad6c1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
@@ -281,4 +281,13 @@ public abstract class AbstractFileMergeOperator
   return null;
 }
   }
+
+  @Override
+  public String getName() {
+return AbstractFileMergeOperator.getOperatorName();
+  }
+
+  public static String getOperatorName() {
+return "MERGE";
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/12cb6a98/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/AppMasterEventOperator.java

hive git commit: HIVE-12741: HS2 ShutdownHookManager holds extra of Driver instance in master/branch-2.0 (Thejas Nair, reviewed by Daniel Dai)

2016-04-07 Thread daijy

Repository: hive
Updated Branches:
  refs/heads/branch-1 4ac966cd8 -> bcbc41526


HIVE-12741: HS2 ShutdownHookManager holds extra of Driver instance in 
master/branch-2.0 (Thejas Nair, reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bcbc4152
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bcbc4152
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bcbc4152

Branch: refs/heads/branch-1
Commit: bcbc41526b9562a406899731f62bf55825130c6e
Parents: 4ac966c
Author: Daniel Dai 
Authored: Thu Apr 7 14:03:01 2016 -0700
Committer: Daniel Dai 
Committed: Thu Apr 7 14:03:01 2016 -0700

--
 .../java/org/apache/hive/common/util/ShutdownHookManager.java | 5 -
 ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 7 ---
 2 files changed, 8 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/bcbc4152/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
--
diff --git 
a/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java 
b/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
index 0392eb5..65f7f52 100644
--- a/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
+++ b/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
@@ -172,9 +172,12 @@ public class ShutdownHookManager {
*
* @param shutdownHook shutdownHook to remove.
* @return TRUE if the shutdownHook was registered and removed,
-   * FALSE otherwise.
+   * FALSE otherwise (including when shutdownHook == null)
*/
   public static boolean removeShutdownHook(Runnable shutdownHook) {
+if (shutdownHook == null) {
+  return false;
+}
 return MGR.removeShutdownHookInternal(shutdownHook);
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/bcbc4152/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index b2bc71e..5014f1d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -407,6 +407,9 @@ public class Driver implements CommandProcessor {
   // Initialize the transaction manager.  This must be done before analyze 
is called.
   final HiveTxnManager txnManager = SessionState.get().initTxnMgr(conf);
   // In case when user Ctrl-C twice to kill Hive CLI JVM, we want to 
release locks
+
+  // if compile is being called multiple times, clear the old shutdownhook
+  ShutdownHookManager.removeShutdownHook(shutdownRunner);
   shutdownRunner = new Runnable() {
 @Override
 public void run() {
@@ -1930,10 +1933,8 @@ public class Driver implements CommandProcessor {
 LOG.warn("Exception when releasing locking in destroy: " +
 e.getMessage());
   }
-  if (shutdownRunner != null) {
-ShutdownHookManager.removeShutdownHook(shutdownRunner);
-  }
 }
+ShutdownHookManager.removeShutdownHook(shutdownRunner);
   }
 
   public org.apache.hadoop.hive.ql.plan.api.Query getQueryPlan() throws 
IOException {

hive git commit: HIVE-13360: Refactoring Hive Authorization (Pengcheng Xiong, reviewed by Ashutosh Chauhan)

2016-04-07 Thread pxiong

Repository: hive
Updated Branches:
  refs/heads/master 4f9194d16 -> 7e0b08c1d


HIVE-13360: Refactoring Hive Authorization (Pengcheng Xiong, reviewed by 
Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7e0b08c1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7e0b08c1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7e0b08c1

Branch: refs/heads/master
Commit: 7e0b08c1d1b05ae647921ecf7ea4998908ed9c66
Parents: 4f9194d
Author: Pengcheng Xiong 
Authored: Thu Apr 7 13:38:50 2016 -0700
Committer: Pengcheng Xiong 
Committed: Thu Apr 7 13:38:50 2016 -0700

--
 .../TestHiveAuthorizerCheckInvocation.java  |  2 +-
 .../plugin/TestHiveAuthorizerShowFilters.java   |  4 +-
 .../jdbc/authorization/TestHS2AuthzContext.java | 14 ++--
 .../authorization/TestJdbcMetadataApiAuth.java  |  4 +-
 .../hive/ql/security/DummyAuthenticator.java|  5 ++
 .../security/InjectableDummyAuthenticator.java  |  5 ++
 ...SQLStdHiveAuthorizationValidatorForTest.java | 47 ++-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |  5 +-
 .../hadoop/hive/ql/parse/MaskAndFilterInfo.java | 33 
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  | 36 ++---
 .../apache/hadoop/hive/ql/parse/TableMask.java  | 78 ++
 .../hadoop/hive/ql/processors/CommandUtil.java  |  5 +-
 .../ql/security/HadoopDefaultAuthenticator.java |  5 ++
 .../ql/security/HiveAuthenticationProvider.java |  2 +
 .../SessionStateConfigUserAuthenticator.java|  5 ++
 .../security/SessionStateUserAuthenticator.java |  5 ++
 .../AuthorizationMetaStoreFilterHook.java   |  3 +-
 .../plugin/HiveAuthorizationValidator.java  | 12 +--
 .../authorization/plugin/HiveAuthorizer.java| 68 +---
 .../plugin/HiveAuthorizerImpl.java  | 20 ++---
 .../authorization/plugin/HiveAuthzContext.java  | 83 
 .../plugin/HivePrivilegeObject.java | 31 
 .../authorization/plugin/HiveV1Authorizer.java  | 20 +
 .../authorization/plugin/QueryContext.java  | 61 ++
 .../sqlstd/DummyHiveAuthorizationValidator.java | 21 ++---
 .../SQLStdHiveAuthorizationValidator.java   | 22 ++
 .../cli/operation/MetadataOperation.java|  5 +-
 27 files changed, 315 insertions(+), 286 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/7e0b08c1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
index 9f47f84..acf2663 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
@@ -373,7 +373,7 @@ public class TestHiveAuthorizerCheckInvocation {
 
 verify(mockedAuthorizer).checkPrivileges(any(HiveOperationType.class),
 inputsCapturer.capture(), outputsCapturer.capture(),
-any(HiveAuthzContext.class));
+any(QueryContext.class));
 
 return new ImmutablePair(inputsCapturer.getValue(), 
outputsCapturer.getValue());
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/7e0b08c1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
index 5922a8c..0209044 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
@@ -77,7 +77,7 @@ public class TestHiveAuthorizerShowFilters {
 protected abstract class AuthorizerWithFilterCmdImpl implements 
HiveAuthorizer {
   @Override
   public List 
filterListCmdObjects(List listObjs,
-  HiveAuthzContext context) throws HiveAuthzPluginException, 
HiveAccessControlException {
+  QueryContext context) throws HiveAuthzPluginException, 
HiveAccessControlException {
 // capture arguments in static

svn commit: r985011 - in /websites/staging/hive/trunk/content: ./ people.html

2016-04-07 Thread buildbot

Author: buildbot
Date: Thu Apr  7 14:32:30 2016
New Revision: 985011

Log:
Staging update by buildbot for hive

Modified:
websites/staging/hive/trunk/content/   (props changed)
websites/staging/hive/trunk/content/people.html

Propchange: websites/staging/hive/trunk/content/
--
--- cms:source-revision (original)
+++ cms:source-revision Thu Apr  7 14:32:30 2016
@@ -1 +1 @@
-1737450
+1738134

Modified: websites/staging/hive/trunk/content/people.html
==
--- websites/staging/hive/trunk/content/people.html (original)
+++ websites/staging/hive/trunk/content/people.html Thu Apr  7 14:32:30 2016
@@ -600,6 +600,17 @@ tr:nth-child(2n+1) {
 http://www.epam.com/;>EPAM 
 
 
+
+aihuaxu 
+Aihua Xu 
+http://cloudera.com/;>Cloudera 
+
+
+
+ychena 
+Yongzhi Chen 
+http://cloudera.com/;>Cloudera 
+
 
 PMC members are also Hive committers.

svn commit: r1738134 - /hive/cms/trunk/content/people.mdtext

2016-04-07 Thread aihuaxu

Author: aihuaxu
Date: Thu Apr  7 14:32:25 2016
New Revision: 1738134

URL: http://svn.apache.org/viewvc?rev=1738134=rev
Log:
HIVE-13427: Update committer list (Reviewed by Szehon Ho)

Modified:
hive/cms/trunk/content/people.mdtext

Modified: hive/cms/trunk/content/people.mdtext
URL: 
http://svn.apache.org/viewvc/hive/cms/trunk/content/people.mdtext?rev=1738134=1738133=1738134=diff
==
--- hive/cms/trunk/content/people.mdtext (original)
+++ hive/cms/trunk/content/people.mdtext Thu Apr  7 14:32:25 2016
@@ -482,6 +482,17 @@ tr:nth-child(2n+1) {
 http://www.epam.com/;>EPAM 
 
 
+
+aihuaxu 
+Aihua Xu 
+http://cloudera.com/;>Cloudera 
+
+
+
+ychena 
+Yongzhi Chen 
+http://cloudera.com/;>Cloudera 
+
 
 PMC members are also Hive committers.

hive git commit: HIVE-13333: StatsOptimizer throws ClassCastException (Pengcheng Xiong, reviewed by Ashutosh Chauhan)

hive git commit: HIVE-13417: Some vector operators return 'OP' as name (Gunther Hagleitner, reviewed by Vikram Dixit K)

hive git commit: HIVE-12741: HS2 ShutdownHookManager holds extra of Driver instance in master/branch-2.0 (Thejas Nair, reviewed by Daniel Dai)

hive git commit: HIVE-13360: Refactoring Hive Authorization (Pengcheng Xiong, reviewed by Ashutosh Chauhan)

svn commit: r985011 - in /websites/staging/hive/trunk/content: ./ people.html

svn commit: r1738134 - /hive/cms/trunk/content/people.mdtext

6 matches

Site Navigation

Mail list logo

Footer information