hive git commit: HIVE-12674: HS2 Tez sessions should have maximum age (Sergey Shelukhin, reviewed by Vikram Dixit K)

2015-12-24 Thread gunther 
Repository: hive
Updated Branches:
  refs/heads/master b9a17486d -> 6e513b06c


HIVE-12674: HS2 Tez sessions should have maximum age (Sergey Shelukhin, 
reviewed by Vikram Dixit K)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6e513b06
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6e513b06
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6e513b06

Branch: refs/heads/master
Commit: 6e513b06c12e508af655a6bea4aef55b86619cc6
Parents: b9a1748
Author: Gunther Hagleitner 
Authored: Thu Dec 24 12:48:31 2015 -0800
Committer: Gunther Hagleitner 
Committed: Thu Dec 24 12:48:31 2015 -0800

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  10 +-
 .../hadoop/hive/ql/exec/mr/ExecDriver.java  |   2 +-
 .../hadoop/hive/ql/exec/tez/TezJobMonitor.java  |   2 +-
 .../hive/ql/exec/tez/TezSessionPoolManager.java | 384 +--
 .../hive/ql/exec/tez/TezSessionState.java   |  48 ++-
 .../hadoop/hive/ql/session/SessionState.java|   2 +-
 .../hive/ql/exec/tez/SampleTezSessionState.java |   8 +-
 .../hive/ql/exec/tez/TestTezSessionPool.java|  11 +-
 8 files changed, 402 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/6e513b06/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d2dd9c6..591c0ab 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1885,10 +1885,18 @@ public class HiveConf extends Configuration {
 "A positive integer that determines the number of Tez sessions that 
should be\n" +
 "launched on each of the queues specified by 
\"hive.server2.tez.default.queues\".\n" +
 "Determines the parallelism on each queue."),
-
HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS("hive.server2.tez.initialize.default.sessions",
 false,
+
HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS("hive.server2.tez.initialize.default.sessions",
+false,
 "This flag is used in HiveServer2 to enable a user to use HiveServer2 
without\n" +
 "turning on Tez for HiveServer2. The user could potentially want to 
run queries\n" +
 "over Tez without the pool of sessions."),
+HIVE_SERVER2_TEZ_SESSION_LIFETIME("hive.server2.tez.session.lifetime", 
"162h",
+new TimeValidator(TimeUnit.HOURS),
+"The lifetime of the Tez sessions launched by HS2 when default 
sessions are enabled.\n" +
+"Set to 0 to disable session expiration."),
+
HIVE_SERVER2_TEZ_SESSION_LIFETIME_JITTER("hive.server2.tez.session.lifetime.jitter",
 "3h",
+new TimeValidator(TimeUnit.HOURS),
+"The jitter for Tez session lifetime; prevents all the sessions from 
restarting at once."),
 
 // Operation log configuration
 
HIVE_SERVER2_LOGGING_OPERATION_ENABLED("hive.server2.logging.operation.enabled",
 true,

http://git-wip-us.apache.org/repos/asf/hive/blob/6e513b06/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index 971dac9..ab7fd93 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -423,7 +423,7 @@ public class ExecDriver extends Task implements 
Serializable, Hadoop
   if (HiveConf.getVar(job, 
HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")
   && ss != null) {
 TezSessionState session = ss.getTezSession();
-TezSessionPoolManager.getInstance().close(session, true);
+TezSessionPoolManager.getInstance().closeIfNotDefault(session, true);
   }
 
   // Finally SUBMIT the JOB!

http://git-wip-us.apache.org/repos/asf/hive/blob/6e513b06/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
index f6bc19c..e81b73d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
@@ -140,7 +140,7 @@ public class TezJobMonitor {
 try {
   for (TezSessionState s : 
TezSessionPoolManager.getInstance().getOpenSessions()) {
 System.err.println("Shutting down tez session.");
-

hive git commit: HIVE-12740: NPE with HS2 when using null input format (Vikram Dixit K via Gunther Hagleitner)

2015-12-24 Thread gunther 
Repository: hive
Updated Branches:
  refs/heads/branch-2.0 3bc74bb1f -> 5c66ed670


HIVE-12740: NPE with HS2 when using null input format (Vikram Dixit K via 
Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5c66ed67
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5c66ed67
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5c66ed67

Branch: refs/heads/branch-2.0
Commit: 5c66ed670e93eece2b0d52ce24e523430be0a97f
Parents: 3bc74bb
Author: Gunther Hagleitner 
Authored: Thu Dec 24 13:00:47 2015 -0800
Committer: Gunther Hagleitner 
Committed: Thu Dec 24 13:04:48 2015 -0800

--
 .../org/apache/hadoop/hive/ql/exec/Utilities.java  | 17 -
 .../apache/hadoop/hive/ql/exec/tez/DagUtils.java   |  9 +
 2 files changed, 21 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/5c66ed67/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 9491015..fce11c8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -215,6 +215,7 @@ public final class Utilities {
   public static final String MAPRED_MAPPER_CLASS = "mapred.mapper.class";
   public static final String MAPRED_REDUCER_CLASS = "mapred.reducer.class";
   public static final String HIVE_ADDED_JARS = "hive.added.jars";
+  public static final String VECTOR_MODE = "VECTOR_MODE";
   public static String MAPNAME = "Map ";
   public static String REDUCENAME = "Reducer ";
 
@@ -3238,12 +3239,18 @@ public final class Utilities {
* but vectorization disallowed eg. for FetchOperator execution.
*/
   public static boolean isVectorMode(Configuration conf) {
-if (HiveConf.getBoolVar(conf, 
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED) &&
-Utilities.getPlanPath(conf) != null && Utilities
-.getMapWork(conf).getVectorMode()) {
-  return true;
+if (conf.get(VECTOR_MODE) != null) {
+  // this code path is necessary, because with HS2 and client
+  // side split generation we end up not finding the map work.
+  // This is because of thread local madness (tez split
+  // generation is multi-threaded - HS2 plan cache uses thread
+  // locals).
+  return conf.getBoolean(VECTOR_MODE, false);
+} else {
+  return HiveConf.getBoolVar(conf, 
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)
+&& Utilities.getPlanPath(conf) != null
+&& Utilities.getMapWork(conf).getVectorMode();
 }
-return false;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/5c66ed67/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 6e196e6..e8864ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -614,6 +614,15 @@ public class DagUtils {
   }
 } else {
   // Setup client side split generation.
+
+  // we need to set this, because with HS2 and client side split
+  // generation we end up not finding the map work. This is
+  // because of thread local madness (tez split generation is
+  // multi-threaded - HS2 plan cache uses thread locals). Setting
+  // VECTOR_MODE causes the split gen code to use the conf instead
+  // of the map work.
+  conf.setBoolean(Utilities.VECTOR_MODE, mapWork.getVectorMode());
+
   dataSource = 
MRInputHelpers.configureMRInputWithLegacySplitGeneration(conf, new Path(tezDir,
   "split_" + mapWork.getName().replaceAll(" ", "_")), true);
   numTasks = dataSource.getNumberOfShards();

hive git commit: HIVE-12684: NPE in stats annotation when all values in decimal column are NULLs (Prasanth Jayachandran reviewed by Pengcheng Xiong)

2015-12-24 Thread prasanthj 
Repository: hive
Updated Branches:
  refs/heads/branch-1 11b0415e4 -> 1a159bc1b


HIVE-12684: NPE in stats annotation when all values in decimal column are NULLs 
(Prasanth Jayachandran reviewed by Pengcheng Xiong)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1a159bc1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1a159bc1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1a159bc1

Branch: refs/heads/branch-1
Commit: 1a159bc1b3bc4490bba9dc149e6de7feb9277710
Parents: 11b0415
Author: Prasanth Jayachandran 
Authored: Thu Dec 17 13:38:57 2015 -0600
Committer: Prasanth Jayachandran 
Committed: Thu Dec 24 15:23:04 2015 -0600

--
 .../apache/hadoop/hive/ql/stats/StatsUtils.java |  20 ++--
 .../test/queries/clientpositive/decimal_stats.q |  16 +++
 .../results/clientpositive/decimal_stats.q.out  | 106 +++
 3 files changed, 135 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1a159bc1/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index f67ddf1..1e4c85e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -692,13 +692,19 @@ public class StatsUtils {
   cs.setAvgColLen(JavaDataModel.get().lengthOfDecimal());
   cs.setCountDistint(csd.getDecimalStats().getNumDVs());
   cs.setNumNulls(csd.getDecimalStats().getNumNulls());
-  Decimal val = csd.getDecimalStats().getHighValue();
-  BigDecimal maxVal = HiveDecimal.
-  create(new BigInteger(val.getUnscaled()), 
val.getScale()).bigDecimalValue();
-  val = csd.getDecimalStats().getLowValue();
-  BigDecimal minVal = HiveDecimal.
-  create(new BigInteger(val.getUnscaled()), 
val.getScale()).bigDecimalValue();
-  cs.setRange(minVal, maxVal);
+  Decimal highValue = csd.getDecimalStats().getHighValue();
+  Decimal lowValue = csd.getDecimalStats().getLowValue();
+  if (highValue != null && highValue.getUnscaled() != null
+  && lowValue != null && lowValue.getUnscaled() != null) {
+HiveDecimal maxHiveDec = HiveDecimal.create(new 
BigInteger(highValue.getUnscaled()), highValue.getScale());
+BigDecimal maxVal = maxHiveDec == null ? null : 
maxHiveDec.bigDecimalValue();
+HiveDecimal minHiveDec = HiveDecimal.create(new 
BigInteger(lowValue.getUnscaled()), lowValue.getScale());
+BigDecimal minVal = minHiveDec == null ? null : 
minHiveDec.bigDecimalValue();
+
+if (minVal != null && maxVal != null) {
+  cs.setRange(minVal, maxVal);
+}
+  }
 } else if (colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) {
   cs.setAvgColLen(JavaDataModel.get().lengthOfDate());
 } else {

http://git-wip-us.apache.org/repos/asf/hive/blob/1a159bc1/ql/src/test/queries/clientpositive/decimal_stats.q
--
diff --git a/ql/src/test/queries/clientpositive/decimal_stats.q 
b/ql/src/test/queries/clientpositive/decimal_stats.q
new file mode 100644
index 000..2370e7d
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/decimal_stats.q
@@ -0,0 +1,16 @@
+set hive.stats.fetch.column.stats=true;
+drop table if exists decimal_1;
+
+create table decimal_1 (t decimal(4,2), u decimal(5), v decimal);
+
+desc decimal_1;
+
+insert overwrite table decimal_1
+  select cast('17.29' as decimal(4,2)), 3.1415926BD, null from src;
+
+analyze table decimal_1 compute statistics for columns;
+
+desc formatted decimal_1 v;
+
+explain select * from decimal_1 order by 1 limit 100;
+drop table decimal_1;

http://git-wip-us.apache.org/repos/asf/hive/blob/1a159bc1/ql/src/test/results/clientpositive/decimal_stats.q.out
--
diff --git a/ql/src/test/results/clientpositive/decimal_stats.q.out 
b/ql/src/test/results/clientpositive/decimal_stats.q.out
new file mode 100644
index 000..dabf7f8
--- /dev/null
+++ b/ql/src/test/results/clientpositive/decimal_stats.q.out
@@ -0,0 +1,106 @@
+PREHOOK: query: drop table if exists decimal_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists decimal_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table decimal_1 (t decimal(4,2), u decimal(5), v 
decimal)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@decimal_1
+POSTHOOK: query: create table decimal_1 (t decimal(4,2), u decimal(5), v 
decimal)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output:

hive git commit: HIVE-12738: subquery with NOT IN failing due to ClassCastException (Matt McCline via Gunther Hagleitner)

2015-12-24 Thread gunther 
Repository: hive
Updated Branches:
  refs/heads/master 267b4e811 -> 1753630fe


HIVE-12738: subquery with NOT IN failing due to ClassCastException (Matt 
McCline via Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1753630f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1753630f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1753630f

Branch: refs/heads/master
Commit: 1753630fede17b841be3ed476121339ab7bda21a
Parents: 267b4e8
Author: Gunther Hagleitner 
Authored: Thu Dec 24 13:32:17 2015 -0800
Committer: Gunther Hagleitner 
Committed: Thu Dec 24 13:56:15 2015 -0800

--
 .../test/resources/testconfiguration.properties |   1 +
 .../hive/ql/optimizer/physical/Vectorizer.java  | 111 
 .../clientpositive/vector_groupby_mapjoin.q |  22 +++
 .../tez/vector_groupby_mapjoin.q.out| 125 ++
 .../clientpositive/vector_groupby_mapjoin.q.out | 167 +++
 5 files changed, 389 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1753630f/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index 3efc1ac..2d8117f 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -264,6 +264,7 @@ minitez.query.files.shared=acid_globallimit.q,\
   vector_distinct_2.q,\
   vector_elt.q,\
   vector_groupby_3.q,\
+  vector_groupby_mapjoin.q,\
   vector_groupby_reduce.q,\
   vector_grouping_sets.q,\
   vector_if_expr.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/1753630f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index a842649..1629a5d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -338,6 +338,8 @@ public class Vectorizer implements PhysicalPlanResolver {
 
 String[] scratchTypeNameArray;
 
+Set nonVectorizedOps;
+
 VectorTaskColumnInfo() {
   partitionColumnCount = 0;
 }
@@ -355,6 +357,14 @@ public class Vectorizer implements PhysicalPlanResolver {
   this.scratchTypeNameArray = scratchTypeNameArray;
 }
 
+public void setNonVectorizedOps(Set 
nonVectorizedOps) {
+  this.nonVectorizedOps = nonVectorizedOps;
+}
+
+public Set getNonVectorizedOps() {
+  return nonVectorizedOps;
+}
+
 public void transferToBaseWork(BaseWork baseWork) {
 
   String[] columnNameArray = columnNames.toArray(new String[0]);
@@ -701,6 +711,7 @@ public class Vectorizer implements PhysicalPlanResolver {
   }
 }
   }
+  vectorTaskColumnInfo.setNonVectorizedOps(vnp.getNonVectorizedOps());
   return true;
 }
 
@@ -819,6 +830,7 @@ public class Vectorizer implements PhysicalPlanResolver {
   }
 }
   }
+  vectorTaskColumnInfo.setNonVectorizedOps(vnp.getNonVectorizedOps());
   return true;
 }
 
@@ -863,6 +875,14 @@ public class Vectorizer implements PhysicalPlanResolver {
 private final MapWork mapWork;
 private final boolean isTez;
 
+// Children of Vectorized GROUPBY that outputs rows instead of vectorized 
row batchs.
+protected final Set nonVectorizedOps =
+new HashSet();
+
+public Set getNonVectorizedOps() {
+  return nonVectorizedOps;
+}
+
 public MapWorkValidationNodeProcessor(MapWork mapWork, boolean isTez) {
   this.mapWork = mapWork;
   this.isTez = isTez;
@@ -873,7 +893,7 @@ public class Vectorizer implements PhysicalPlanResolver {
 Object... nodeOutputs) throws SemanticException {
   for (Node n : stack) {
 Operator op = (Operator) n;
-if (nonVectorizableChildOfGroupBy(op)) {
+if (nonVectorizedOps.contains(op)) {
   return new Boolean(true);
 }
 boolean ret;
@@ -886,6 +906,12 @@ public class Vectorizer implements PhysicalPlanResolver {
   LOG.info("MapWork Operator: " + op.getName() + " could not be 
vectorized.");
   return new Boolean(false);
 }
+// When Vectorized GROUPBY outputs rows instead of vectorized row 
batches, we don't
+// vectorize the operators below it.
+if (isVectorizedGroupByThatOutputsRows(op)) {

hive git commit: HIVE-12741: HS2 ShutdownHookManager holds extra of Driver instance in master/branch-2.0 (Thejas Nair, reviewed by Daniel Dai)

2015-12-24 Thread gunther 
Repository: hive
Updated Branches:
  refs/heads/branch-2.0 5c66ed670 -> 237729430


HIVE-12741: HS2 ShutdownHookManager holds extra of Driver instance in 
master/branch-2.0 (Thejas Nair, reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/23772943
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/23772943
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/23772943

Branch: refs/heads/branch-2.0
Commit: 23772943093affc49eeda5655205c6ae9cc2d26c
Parents: 5c66ed6
Author: Gunther Hagleitner 
Authored: Thu Dec 24 13:18:59 2015 -0800
Committer: Gunther Hagleitner 
Committed: Thu Dec 24 13:19:39 2015 -0800

--
 .../java/org/apache/hive/common/util/ShutdownHookManager.java  | 5 -
 ql/src/java/org/apache/hadoop/hive/ql/Driver.java  | 6 +++---
 2 files changed, 7 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/23772943/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
--
diff --git 
a/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java 
b/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
index b5f7e69..5efeda9 100644
--- a/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
+++ b/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
@@ -172,9 +172,12 @@ public class ShutdownHookManager {
*
* @param shutdownHook shutdownHook to remove.
* @return TRUE if the shutdownHook was registered and removed,
-   * FALSE otherwise.
+   * FALSE otherwise (including when shutdownHook == null)
*/
   public static boolean removeShutdownHook(Runnable shutdownHook) {
+if (shutdownHook == null) {
+  return false;
+}
 return MGR.removeShutdownHookInternal(shutdownHook);
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/23772943/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 2deeb48..485e901 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -420,6 +420,8 @@ public class Driver implements CommandProcessor {
   final HiveTxnManager txnManager = SessionState.get().initTxnMgr(conf);
   // In case when user Ctrl-C twice to kill Hive CLI JVM, we want to 
release locks
 
+  // if compile is being called multiple times, clear the old shutdownhook
+  ShutdownHookManager.removeShutdownHook(shutdownRunner);
   shutdownRunner = new Runnable() {
 @Override
 public void run() {
@@ -1955,10 +1957,8 @@ public class Driver implements CommandProcessor {
 LOG.warn("Exception when releasing locking in destroy: " +
 e.getMessage());
   }
-  if (shutdownRunner != null) {
-ShutdownHookManager.removeShutdownHook(shutdownRunner);
-  }
 }
+ShutdownHookManager.removeShutdownHook(shutdownRunner);
   }
 
   public org.apache.hadoop.hive.ql.plan.api.Query getQueryPlan() throws 
IOException {

hive git commit: HIVE-12738: subquery with NOT IN failing due to ClassCastException (Matt McCline via Gunther Hagleitner)

2015-12-24 Thread gunther 
Repository: hive
Updated Branches:
  refs/heads/branch-2.0 237729430 -> 55c629691


HIVE-12738: subquery with NOT IN failing due to ClassCastException (Matt 
McCline via Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/55c62969
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/55c62969
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/55c62969

Branch: refs/heads/branch-2.0
Commit: 55c6296914c63d290a7db332eb4c55ab3397691a
Parents: 2377294
Author: Gunther Hagleitner 
Authored: Thu Dec 24 13:32:17 2015 -0800
Committer: Gunther Hagleitner 
Committed: Thu Dec 24 14:14:11 2015 -0800

--
 .../test/resources/testconfiguration.properties |   1 +
 .../hive/ql/optimizer/physical/Vectorizer.java  | 111 
 .../clientpositive/vector_groupby_mapjoin.q |  22 +++
 .../tez/vector_groupby_mapjoin.q.out| 125 ++
 .../clientpositive/vector_groupby_mapjoin.q.out | 167 +++
 5 files changed, 389 insertions(+), 37 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/55c62969/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index 1e7dce3..fd48cf4 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -264,6 +264,7 @@ minitez.query.files.shared=acid_globallimit.q,\
   vector_distinct_2.q,\
   vector_elt.q,\
   vector_groupby_3.q,\
+  vector_groupby_mapjoin.q,\
   vector_groupby_reduce.q,\
   vector_grouping_sets.q,\
   vector_if_expr.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/55c62969/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index a842649..1629a5d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -338,6 +338,8 @@ public class Vectorizer implements PhysicalPlanResolver {
 
 String[] scratchTypeNameArray;
 
+Set nonVectorizedOps;
+
 VectorTaskColumnInfo() {
   partitionColumnCount = 0;
 }
@@ -355,6 +357,14 @@ public class Vectorizer implements PhysicalPlanResolver {
   this.scratchTypeNameArray = scratchTypeNameArray;
 }
 
+public void setNonVectorizedOps(Set 
nonVectorizedOps) {
+  this.nonVectorizedOps = nonVectorizedOps;
+}
+
+public Set getNonVectorizedOps() {
+  return nonVectorizedOps;
+}
+
 public void transferToBaseWork(BaseWork baseWork) {
 
   String[] columnNameArray = columnNames.toArray(new String[0]);
@@ -701,6 +711,7 @@ public class Vectorizer implements PhysicalPlanResolver {
   }
 }
   }
+  vectorTaskColumnInfo.setNonVectorizedOps(vnp.getNonVectorizedOps());
   return true;
 }
 
@@ -819,6 +830,7 @@ public class Vectorizer implements PhysicalPlanResolver {
   }
 }
   }
+  vectorTaskColumnInfo.setNonVectorizedOps(vnp.getNonVectorizedOps());
   return true;
 }
 
@@ -863,6 +875,14 @@ public class Vectorizer implements PhysicalPlanResolver {
 private final MapWork mapWork;
 private final boolean isTez;
 
+// Children of Vectorized GROUPBY that outputs rows instead of vectorized 
row batchs.
+protected final Set nonVectorizedOps =
+new HashSet();
+
+public Set getNonVectorizedOps() {
+  return nonVectorizedOps;
+}
+
 public MapWorkValidationNodeProcessor(MapWork mapWork, boolean isTez) {
   this.mapWork = mapWork;
   this.isTez = isTez;
@@ -873,7 +893,7 @@ public class Vectorizer implements PhysicalPlanResolver {
 Object... nodeOutputs) throws SemanticException {
   for (Node n : stack) {
 Operator op = (Operator) n;
-if (nonVectorizableChildOfGroupBy(op)) {
+if (nonVectorizedOps.contains(op)) {
   return new Boolean(true);
 }
 boolean ret;
@@ -886,6 +906,12 @@ public class Vectorizer implements PhysicalPlanResolver {
   LOG.info("MapWork Operator: " + op.getName() + " could not be 
vectorized.");
   return new Boolean(false);
 }
+// When Vectorized GROUPBY outputs rows instead of vectorized row 
batches, we don't
+// vectorize the operators below it.
+if

hive git commit: HIVE-12674: HS2 Tez sessions should have maximum age (Sergey Shelukhin, reviewed by Vikram Dixit K)

2015-12-24 Thread gunther 
Repository: hive
Updated Branches:
  refs/heads/branch-2.0 c53800185 -> 3bc74bb1f


HIVE-12674: HS2 Tez sessions should have maximum age (Sergey Shelukhin, 
reviewed by Vikram Dixit K)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3bc74bb1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3bc74bb1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3bc74bb1

Branch: refs/heads/branch-2.0
Commit: 3bc74bb1fb679d55eef17d554f39b086dd4d5cdd
Parents: c538001
Author: Gunther Hagleitner 
Authored: Thu Dec 24 12:48:31 2015 -0800
Committer: Gunther Hagleitner 
Committed: Thu Dec 24 12:54:04 2015 -0800

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  10 +-
 .../hadoop/hive/ql/exec/mr/ExecDriver.java  |   2 +-
 .../hadoop/hive/ql/exec/tez/TezJobMonitor.java  |   2 +-
 .../hive/ql/exec/tez/TezSessionPoolManager.java | 384 +--
 .../hive/ql/exec/tez/TezSessionState.java   |  48 ++-
 .../hadoop/hive/ql/session/SessionState.java|   2 +-
 .../hive/ql/exec/tez/SampleTezSessionState.java |   8 +-
 .../hive/ql/exec/tez/TestTezSessionPool.java|  11 +-
 8 files changed, 402 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/3bc74bb1/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index b924d20..e3cbb0b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1873,10 +1873,18 @@ public class HiveConf extends Configuration {
 "A positive integer that determines the number of Tez sessions that 
should be\n" +
 "launched on each of the queues specified by 
\"hive.server2.tez.default.queues\".\n" +
 "Determines the parallelism on each queue."),
-
HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS("hive.server2.tez.initialize.default.sessions",
 false,
+
HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS("hive.server2.tez.initialize.default.sessions",
+false,
 "This flag is used in HiveServer2 to enable a user to use HiveServer2 
without\n" +
 "turning on Tez for HiveServer2. The user could potentially want to 
run queries\n" +
 "over Tez without the pool of sessions."),
+HIVE_SERVER2_TEZ_SESSION_LIFETIME("hive.server2.tez.session.lifetime", 
"162h",
+new TimeValidator(TimeUnit.HOURS),
+"The lifetime of the Tez sessions launched by HS2 when default 
sessions are enabled.\n" +
+"Set to 0 to disable session expiration."),
+
HIVE_SERVER2_TEZ_SESSION_LIFETIME_JITTER("hive.server2.tez.session.lifetime.jitter",
 "3h",
+new TimeValidator(TimeUnit.HOURS),
+"The jitter for Tez session lifetime; prevents all the sessions from 
restarting at once."),
 
 // Operation log configuration
 
HIVE_SERVER2_LOGGING_OPERATION_ENABLED("hive.server2.logging.operation.enabled",
 true,

http://git-wip-us.apache.org/repos/asf/hive/blob/3bc74bb1/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index 971dac9..ab7fd93 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -423,7 +423,7 @@ public class ExecDriver extends Task implements 
Serializable, Hadoop
   if (HiveConf.getVar(job, 
HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")
   && ss != null) {
 TezSessionState session = ss.getTezSession();
-TezSessionPoolManager.getInstance().close(session, true);
+TezSessionPoolManager.getInstance().closeIfNotDefault(session, true);
   }
 
   // Finally SUBMIT the JOB!

http://git-wip-us.apache.org/repos/asf/hive/blob/3bc74bb1/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
index f6bc19c..e81b73d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
@@ -140,7 +140,7 @@ public class TezJobMonitor {
 try {
   for (TezSessionState s : 
TezSessionPoolManager.getInstance().getOpenSessions()) {
 System.err.println("Shutting down tez

hive git commit: HIVE-12740: NPE with HS2 when using null input format (Vikram Dixit K via Gunther Hagleitner)

2015-12-24 Thread gunther 
Repository: hive
Updated Branches:
  refs/heads/master 6e513b06c -> 0b810991a


HIVE-12740: NPE with HS2 when using null input format (Vikram Dixit K via 
Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0b810991
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0b810991
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0b810991

Branch: refs/heads/master
Commit: 0b810991a12bb7c8d52f64b781772a1deabcbe53
Parents: 6e513b0
Author: Gunther Hagleitner 
Authored: Thu Dec 24 13:00:47 2015 -0800
Committer: Gunther Hagleitner 
Committed: Thu Dec 24 13:00:47 2015 -0800

--
 .../org/apache/hadoop/hive/ql/exec/Utilities.java  | 17 -
 .../apache/hadoop/hive/ql/exec/tez/DagUtils.java   |  9 +
 2 files changed, 21 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/0b810991/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 9491015..fce11c8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -215,6 +215,7 @@ public final class Utilities {
   public static final String MAPRED_MAPPER_CLASS = "mapred.mapper.class";
   public static final String MAPRED_REDUCER_CLASS = "mapred.reducer.class";
   public static final String HIVE_ADDED_JARS = "hive.added.jars";
+  public static final String VECTOR_MODE = "VECTOR_MODE";
   public static String MAPNAME = "Map ";
   public static String REDUCENAME = "Reducer ";
 
@@ -3238,12 +3239,18 @@ public final class Utilities {
* but vectorization disallowed eg. for FetchOperator execution.
*/
   public static boolean isVectorMode(Configuration conf) {
-if (HiveConf.getBoolVar(conf, 
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED) &&
-Utilities.getPlanPath(conf) != null && Utilities
-.getMapWork(conf).getVectorMode()) {
-  return true;
+if (conf.get(VECTOR_MODE) != null) {
+  // this code path is necessary, because with HS2 and client
+  // side split generation we end up not finding the map work.
+  // This is because of thread local madness (tez split
+  // generation is multi-threaded - HS2 plan cache uses thread
+  // locals).
+  return conf.getBoolean(VECTOR_MODE, false);
+} else {
+  return HiveConf.getBoolVar(conf, 
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)
+&& Utilities.getPlanPath(conf) != null
+&& Utilities.getMapWork(conf).getVectorMode();
 }
-return false;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/0b810991/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 6e196e6..e8864ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -614,6 +614,15 @@ public class DagUtils {
   }
 } else {
   // Setup client side split generation.
+
+  // we need to set this, because with HS2 and client side split
+  // generation we end up not finding the map work. This is
+  // because of thread local madness (tez split generation is
+  // multi-threaded - HS2 plan cache uses thread locals). Setting
+  // VECTOR_MODE causes the split gen code to use the conf instead
+  // of the map work.
+  conf.setBoolean(Utilities.VECTOR_MODE, mapWork.getVectorMode());
+
   dataSource = 
MRInputHelpers.configureMRInputWithLegacySplitGeneration(conf, new Path(tezDir,
   "split_" + mapWork.getName().replaceAll(" ", "_")), true);
   numTasks = dataSource.getNumberOfShards();

hive git commit: HIVE-12741: HS2 ShutdownHookManager holds extra of Driver instance in master/branch-2.0 (Thejas Nair, reviewed by Daniel Dai)

2015-12-24 Thread gunther 
Repository: hive
Updated Branches:
  refs/heads/master 0b810991a -> 267b4e811


HIVE-12741: HS2 ShutdownHookManager holds extra of Driver instance in 
master/branch-2.0 (Thejas Nair, reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/267b4e81
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/267b4e81
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/267b4e81

Branch: refs/heads/master
Commit: 267b4e81153756ddb9ce6af23cb2d50e8161de08
Parents: 0b81099
Author: Gunther Hagleitner 
Authored: Thu Dec 24 13:18:59 2015 -0800
Committer: Gunther Hagleitner 
Committed: Thu Dec 24 13:18:59 2015 -0800

--
 .../java/org/apache/hive/common/util/ShutdownHookManager.java  | 5 -
 ql/src/java/org/apache/hadoop/hive/ql/Driver.java  | 6 +++---
 2 files changed, 7 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/267b4e81/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
--
diff --git 
a/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java 
b/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
index b5f7e69..5efeda9 100644
--- a/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
+++ b/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
@@ -172,9 +172,12 @@ public class ShutdownHookManager {
*
* @param shutdownHook shutdownHook to remove.
* @return TRUE if the shutdownHook was registered and removed,
-   * FALSE otherwise.
+   * FALSE otherwise (including when shutdownHook == null)
*/
   public static boolean removeShutdownHook(Runnable shutdownHook) {
+if (shutdownHook == null) {
+  return false;
+}
 return MGR.removeShutdownHookInternal(shutdownHook);
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/267b4e81/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 52f5528..29e6315 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -413,6 +413,8 @@ public class Driver implements CommandProcessor {
   final HiveTxnManager txnManager = SessionState.get().initTxnMgr(conf);
   // In case when user Ctrl-C twice to kill Hive CLI JVM, we want to 
release locks
 
+  // if compile is being called multiple times, clear the old shutdownhook
+  ShutdownHookManager.removeShutdownHook(shutdownRunner);
   shutdownRunner = new Runnable() {
 @Override
 public void run() {
@@ -1910,10 +1912,8 @@ public class Driver implements CommandProcessor {
 LOG.warn("Exception when releasing locking in destroy: " +
 e.getMessage());
   }
-  if (shutdownRunner != null) {
-ShutdownHookManager.removeShutdownHook(shutdownRunner);
-  }
 }
+ShutdownHookManager.removeShutdownHook(shutdownRunner);
   }
 
   public org.apache.hadoop.hive.ql.plan.api.Query getQueryPlan() throws 
IOException {

hive git commit: HIVE-12743: RCFileInputFormat needs to be registered with kryo (Prasanth Jayachandran reviewed by Sergey Shelukhin)

2015-12-24 Thread prasanthj 
Repository: hive
Updated Branches:
  refs/heads/master 7e07a278a -> b9a17486d


HIVE-12743: RCFileInputFormat needs to be registered with kryo (Prasanth 
Jayachandran reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b9a17486
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b9a17486
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b9a17486

Branch: refs/heads/master
Commit: b9a17486d741087fb07cb84184d21c7a4599d0b3
Parents: 7e07a27
Author: Prasanth Jayachandran 
Authored: Thu Dec 24 13:49:50 2015 -0600
Committer: Prasanth Jayachandran 
Committed: Thu Dec 24 13:50:20 2015 -0600

--
 .../org/apache/hadoop/hive/ql/exec/SerializationUtilities.java | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b9a17486/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
index 803f492..45f50c3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
+import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc;
 import org.apache.hadoop.hive.ql.plan.BaseWork;
@@ -91,6 +92,7 @@ public class SerializationUtilities {
   kryo.register(StandardConstantMapObjectInspector.class);
   kryo.register(StandardConstantStructObjectInspector.class);
   kryo.register(SequenceFileInputFormat.class);
+  kryo.register(RCFileInputFormat.class);
   kryo.register(HiveSequenceFileOutputFormat.class);
   kryo.register(SparkEdgeProperty.class);
   kryo.register(SparkWork.class);

hive git commit: HIVE-12743: RCFileInputFormat needs to be registered with kryo (Prasanth Jayachandran reviewed by Sergey Shelukhin)

2015-12-24 Thread prasanthj 
Repository: hive
Updated Branches:
  refs/heads/branch-2.0 b81206a72 -> c53800185


HIVE-12743: RCFileInputFormat needs to be registered with kryo (Prasanth 
Jayachandran reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c5380018
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c5380018
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c5380018

Branch: refs/heads/branch-2.0
Commit: c53800185e052324f596a86dff35978f939cc476
Parents: b81206a
Author: Prasanth Jayachandran 
Authored: Thu Dec 24 13:49:50 2015 -0600
Committer: Prasanth Jayachandran 
Committed: Thu Dec 24 13:50:48 2015 -0600

--
 .../org/apache/hadoop/hive/ql/exec/SerializationUtilities.java | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/c5380018/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
--
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
index 803f492..45f50c3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
+import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc;
 import org.apache.hadoop.hive.ql.plan.BaseWork;
@@ -91,6 +92,7 @@ public class SerializationUtilities {
   kryo.register(StandardConstantMapObjectInspector.class);
   kryo.register(StandardConstantStructObjectInspector.class);
   kryo.register(SequenceFileInputFormat.class);
+  kryo.register(RCFileInputFormat.class);
   kryo.register(HiveSequenceFileOutputFormat.class);
   kryo.register(SparkEdgeProperty.class);
   kryo.register(SparkWork.class);

[2/2] hive git commit: HIVE-12728: Apply DDL restrictions for ORC schema evolution (Prasanth Jayachandran reviewed by Matt McCline and Gunther Hagleitner)

2015-12-24 Thread prasanthj 
HIVE-12728: Apply DDL restrictions for ORC schema evolution (Prasanth 
Jayachandran reviewed by Matt McCline and Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7e07a278
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7e07a278
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7e07a278

Branch: refs/heads/master
Commit: 7e07a278ac2843489f96326f2011b9588551e986
Parents: 07dfece
Author: Prasanth Jayachandran 
Authored: Thu Dec 24 11:34:57 2015 -0600
Committer: Prasanth Jayachandran 
Committed: Thu Dec 24 11:34:57 2015 -0600

--
 .../org/apache/hadoop/hive/ql/ErrorMsg.java |   6 +
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |  88 +++-
 .../clientnegative/orc_change_fileformat.q  |   2 +
 .../queries/clientnegative/orc_change_serde.q   |   2 +
 .../clientnegative/orc_reorder_columns1.q   |   2 +
 .../clientnegative/orc_reorder_columns2.q   |   2 +
 .../clientnegative/orc_replace_columns1.q   |   2 +
 .../clientnegative/orc_replace_columns2.q   |   2 +
 .../clientnegative/orc_replace_columns3.q   |   3 +
 .../clientnegative/orc_type_promotion1.q|   2 +
 .../clientnegative/orc_type_promotion2.q|   9 +
 .../clientnegative/orc_type_promotion3.q|   2 +
 .../test/queries/clientpositive/dbtxnmgr_ddl1.q |   1 -
 .../test/queries/clientpositive/load_orc_part.q |  10 -
 .../test/queries/clientpositive/orc_analyze.q   |  14 --
 .../clientpositive/orc_int_type_promotion.q |  12 --
 .../clientpositive/orc_schema_evolution.q   |  39 
 .../vectorization_short_regress.q   |  15 +-
 .../clientnegative/orc_change_fileformat.q.out  |  13 ++
 .../clientnegative/orc_change_serde.q.out   |  13 ++
 .../clientnegative/orc_reorder_columns1.q.out   |  13 ++
 .../clientnegative/orc_reorder_columns2.q.out   |  13 ++
 .../clientnegative/orc_replace_columns.q.out|  13 ++
 .../clientnegative/orc_replace_columns1.q.out   |  13 ++
 .../clientnegative/orc_replace_columns2.q.out   |  13 ++
 .../clientnegative/orc_replace_columns3.q.out   |  21 ++
 .../clientnegative/orc_type_promotion1.q.out|  13 ++
 .../clientnegative/orc_type_promotion2.q.out|  69 ++
 .../clientnegative/orc_type_promotion3.q.out|  13 ++
 .../results/clientpositive/dbtxnmgr_ddl1.q.out  |   9 -
 .../results/clientpositive/load_orc_part.q.out  |  52 -
 .../results/clientpositive/orc_analyze.q.out| 182 
 .../clientpositive/orc_int_type_promotion.q.out |  86 +---
 .../clientpositive/orc_schema_evolution.q.out   | 211 +++
 .../spark/vectorization_short_regress.q.out |  36 +++-
 .../clientpositive/tez/orc_analyze.q.out| 182 
 .../tez/vectorization_short_regress.q.out   |  36 +++-
 .../vectorization_short_regress.q.out   |  36 +++-
 38 files changed, 673 insertions(+), 577 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/7e07a278/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java 
b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index d759739..7585bad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -427,6 +427,12 @@ public enum ErrorMsg {
   //{2} should be lockid
   LOCK_ACQUIRE_TIMEDOUT(10307, "Lock acquisition for {0} timed out after 
{1}ms.  {2}", true),
   COMPILE_LOCK_TIMED_OUT(10308, "Attempt to acquire compile lock timed out.", 
true),
+  CANNOT_CHANGE_SERDE(10309, "Changing SerDe (from {0}) is not supported for 
table {1}. File format may be incompatible", true),
+  CANNOT_CHANGE_FILEFORMAT(10310, "Changing file format (from {0}) is not 
supported for table {1}", true),
+  CANNOT_REORDER_COLUMNS(10311, "Reordering columns is not supported for table 
{0}. SerDe may be incompatible", true),
+  CANNOT_CHANGE_COLUMN_TYPE(10312, "Changing from type {0} to {1} is not 
supported for column {2}. SerDe may be incompatible", true),
+  REPLACE_CANNOT_DROP_COLUMNS(10313, "Replacing columns cannot drop columns 
for table {0}. SerDe may be incompatible", true),
+  REPLACE_UNSUPPORTED_TYPE_CONVERSION(10314, "Replacing columns with 
unsupported type conversion (from {0} to {1}) for column {2}. SerDe may be 
incompatible", true),
   //== 2 range starts here 
//
   SCRIPT_INIT_ERROR(2, "Unable to initialize custom script."),
   SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your 
custom script. "

http://git-wip-us.apache.org/repos/asf/hive/blob/7e07a278/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java

[1/2] hive git commit: HIVE-12728: Apply DDL restrictions for ORC schema evolution (Prasanth Jayachandran reviewed by Matt McCline and Gunther Hagleitner)

2015-12-24 Thread prasanthj 
Repository: hive
Updated Branches:
  refs/heads/master 07dfecee7 -> 7e07a278a


http://git-wip-us.apache.org/repos/asf/hive/blob/7e07a278/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out 
b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
index 4925e65..f34761c 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
@@ -2964,22 +2964,38 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@test_count
  A masked pattern was here 
 0
-PREHOOK: query: create table alltypesnull like alltypesorc
+PREHOOK: query: CREATE TABLE alltypesnull(
+ctinyint TINYINT,
+csmallint SMALLINT,
+cint INT,
+cbigint BIGINT,
+cfloat FLOAT,
+cdouble DOUBLE,
+cstring1 STRING,
+cstring2 STRING,
+ctimestamp1 TIMESTAMP,
+ctimestamp2 TIMESTAMP,
+cboolean1 BOOLEAN,
+cboolean2 BOOLEAN)
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@alltypesnull
-POSTHOOK: query: create table alltypesnull like alltypesorc
+POSTHOOK: query: CREATE TABLE alltypesnull(
+ctinyint TINYINT,
+csmallint SMALLINT,
+cint INT,
+cbigint BIGINT,
+cfloat FLOAT,
+cdouble DOUBLE,
+cstring1 STRING,
+cstring2 STRING,
+ctimestamp1 TIMESTAMP,
+ctimestamp2 TIMESTAMP,
+cboolean1 BOOLEAN,
+cboolean2 BOOLEAN)
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@alltypesnull
-PREHOOK: query: alter table alltypesnull set fileformat textfile
-PREHOOK: type: ALTERTABLE_FILEFORMAT
-PREHOOK: Input: default@alltypesnull
-PREHOOK: Output: default@alltypesnull
-POSTHOOK: query: alter table alltypesnull set fileformat textfile
-POSTHOOK: type: ALTERTABLE_FILEFORMAT
-POSTHOOK: Input: default@alltypesnull
-POSTHOOK: Output: default@alltypesnull
 PREHOOK: query: insert into table alltypesnull select null, null, null, null, 
null, null, null, null, null, null, null, null from alltypesorc
 PREHOOK: type: QUERY
 PREHOOK: Input: default@alltypesorc

http://git-wip-us.apache.org/repos/asf/hive/blob/7e07a278/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
--
diff --git 
a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out 
b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
index 73034dd..78b5d07 100644
--- a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
@@ -2910,22 +2910,38 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@test_count
  A masked pattern was here 
 0
-PREHOOK: query: create table alltypesnull like alltypesorc
+PREHOOK: query: CREATE TABLE alltypesnull(
+ctinyint TINYINT,
+csmallint SMALLINT,
+cint INT,
+cbigint BIGINT,
+cfloat FLOAT,
+cdouble DOUBLE,
+cstring1 STRING,
+cstring2 STRING,
+ctimestamp1 TIMESTAMP,
+ctimestamp2 TIMESTAMP,
+cboolean1 BOOLEAN,
+cboolean2 BOOLEAN)
 PREHOOK: type: CREATETABLE
 PREHOOK: Output: database:default
 PREHOOK: Output: default@alltypesnull
-POSTHOOK: query: create table alltypesnull like alltypesorc
+POSTHOOK: query: CREATE TABLE alltypesnull(
+ctinyint TINYINT,
+csmallint SMALLINT,
+cint INT,
+cbigint BIGINT,
+cfloat FLOAT,
+cdouble DOUBLE,
+cstring1 STRING,
+cstring2 STRING,
+ctimestamp1 TIMESTAMP,
+ctimestamp2 TIMESTAMP,
+cboolean1 BOOLEAN,
+cboolean2 BOOLEAN)
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@alltypesnull
-PREHOOK: query: alter table alltypesnull set fileformat textfile
-PREHOOK: type: ALTERTABLE_FILEFORMAT
-PREHOOK: Input: default@alltypesnull
-PREHOOK: Output: default@alltypesnull
-POSTHOOK: query: alter table alltypesnull set fileformat textfile
-POSTHOOK: type: ALTERTABLE_FILEFORMAT
-POSTHOOK: Input: default@alltypesnull
-POSTHOOK: Output: default@alltypesnull
 PREHOOK: query: insert into table alltypesnull select null, null, null, null, 
null, null, null, null, null, null, null, null from alltypesorc
 PREHOOK: type: QUERY
 PREHOOK: Input: default@alltypesorc

[2/2] hive git commit: HIVE-12728: Apply DDL restrictions for ORC schema evolution (Prasanth Jayachandran reviewed by Matt McCline and Gunther Hagleitner)

2015-12-24 Thread prasanthj 
HIVE-12728: Apply DDL restrictions for ORC schema evolution (Prasanth 
Jayachandran reviewed by Matt McCline and Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b81206a7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b81206a7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b81206a7

Branch: refs/heads/branch-2.0
Commit: b81206a722ca74fcfae6b841a0e504edaa0a09a9
Parents: 1b1892f
Author: Prasanth Jayachandran 
Authored: Thu Dec 24 11:34:57 2015 -0600
Committer: Prasanth Jayachandran 
Committed: Thu Dec 24 11:42:57 2015 -0600

--
 .../org/apache/hadoop/hive/ql/ErrorMsg.java |   9 +
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |  88 +++-
 .../clientnegative/orc_change_fileformat.q  |   2 +
 .../queries/clientnegative/orc_change_serde.q   |   2 +
 .../clientnegative/orc_reorder_columns1.q   |   2 +
 .../clientnegative/orc_reorder_columns2.q   |   2 +
 .../clientnegative/orc_replace_columns1.q   |   2 +
 .../clientnegative/orc_replace_columns2.q   |   2 +
 .../clientnegative/orc_replace_columns3.q   |   3 +
 .../clientnegative/orc_type_promotion1.q|   2 +
 .../clientnegative/orc_type_promotion2.q|   9 +
 .../clientnegative/orc_type_promotion3.q|   2 +
 .../test/queries/clientpositive/dbtxnmgr_ddl1.q |   1 -
 .../test/queries/clientpositive/load_orc_part.q |  10 -
 .../test/queries/clientpositive/orc_analyze.q   |  14 --
 .../clientpositive/orc_int_type_promotion.q |  12 --
 .../clientpositive/orc_schema_evolution.q   |  39 
 .../vectorization_short_regress.q   |  15 +-
 .../clientnegative/orc_change_fileformat.q.out  |  13 ++
 .../clientnegative/orc_change_serde.q.out   |  13 ++
 .../clientnegative/orc_reorder_columns1.q.out   |  13 ++
 .../clientnegative/orc_reorder_columns2.q.out   |  13 ++
 .../clientnegative/orc_replace_columns.q.out|  13 ++
 .../clientnegative/orc_replace_columns1.q.out   |  13 ++
 .../clientnegative/orc_replace_columns2.q.out   |  13 ++
 .../clientnegative/orc_replace_columns3.q.out   |  21 ++
 .../clientnegative/orc_type_promotion1.q.out|  13 ++
 .../clientnegative/orc_type_promotion2.q.out|  69 ++
 .../clientnegative/orc_type_promotion3.q.out|  13 ++
 .../results/clientpositive/dbtxnmgr_ddl1.q.out  |   9 -
 .../results/clientpositive/load_orc_part.q.out  |  52 -
 .../results/clientpositive/orc_analyze.q.out| 182 
 .../clientpositive/orc_int_type_promotion.q.out |  86 +---
 .../clientpositive/orc_schema_evolution.q.out   | 211 +++
 .../spark/vectorization_short_regress.q.out |  36 +++-
 .../clientpositive/tez/orc_analyze.q.out| 182 
 .../tez/vectorization_short_regress.q.out   |  36 +++-
 .../vectorization_short_regress.q.out   |  36 +++-
 38 files changed, 676 insertions(+), 577 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b81206a7/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java 
b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 8a47605..7585bad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -424,6 +424,15 @@ public enum ErrorMsg {
   CTAS_LOCATION_NONEMPTY(10304, "CREATE-TABLE-AS-SELECT cannot create table 
with location to a non-empty directory."),
   CTAS_CREATES_VOID_TYPE(10305, "CREATE-TABLE-AS-SELECT creates a VOID type, 
please use CAST to specify the type, near field: "),
   TBL_SORTED_NOT_BUCKETED(10306, "Destination table {0} found to be sorted but 
not bucketed.", true),
+  //{2} should be lockid
+  LOCK_ACQUIRE_TIMEDOUT(10307, "Lock acquisition for {0} timed out after 
{1}ms.  {2}", true),
+  COMPILE_LOCK_TIMED_OUT(10308, "Attempt to acquire compile lock timed out.", 
true),
+  CANNOT_CHANGE_SERDE(10309, "Changing SerDe (from {0}) is not supported for 
table {1}. File format may be incompatible", true),
+  CANNOT_CHANGE_FILEFORMAT(10310, "Changing file format (from {0}) is not 
supported for table {1}", true),
+  CANNOT_REORDER_COLUMNS(10311, "Reordering columns is not supported for table 
{0}. SerDe may be incompatible", true),
+  CANNOT_CHANGE_COLUMN_TYPE(10312, "Changing from type {0} to {1} is not 
supported for column {2}. SerDe may be incompatible", true),
+  REPLACE_CANNOT_DROP_COLUMNS(10313, "Replacing columns cannot drop columns 
for table {0}. SerDe may be incompatible", true),
+  REPLACE_UNSUPPORTED_TYPE_CONVERSION(10314, "Replacing columns with 
unsupported type conversion (from {0} to {1}) for column {2}. SerDe may be 
incompatible", true),