hive git commit: HIVE-14799: Query operation are not thread safe during its cancellation (Chaoyu Tang, reviewed by Sergey Shelukhin, Yongzhi Chen)

2016-10-15 Thread ctang 
Repository: hive
Updated Branches:
  refs/heads/master c71ef4fed -> 1901e3a6a


HIVE-14799: Query operation are not thread safe during its cancellation (Chaoyu 
Tang, reviewed by Sergey Shelukhin, Yongzhi Chen)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1901e3a6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1901e3a6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1901e3a6

Branch: refs/heads/master
Commit: 1901e3a6ab97c150905c04c591b33b2c640e4b87
Parents: c71ef4f
Author: ctang 
Authored: Sat Oct 15 08:55:36 2016 -0400
Committer: ctang 
Committed: Sat Oct 15 08:55:36 2016 -0400

--
 .../java/org/apache/hadoop/hive/ql/Driver.java  | 665 +--
 1 file changed, 468 insertions(+), 197 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/1901e3a6/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java 
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index dd55434..9e5fd37 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -164,8 +164,6 @@ public class Driver implements CommandProcessor {
   private int maxthreads;
   private int tryCount = Integer.MAX_VALUE;
 
-  private boolean destroyed;
-
   private String userName;
 
   // HS2 operation handle guid string
@@ -180,6 +178,28 @@ public class Driver implements CommandProcessor {
   // Query hooks that execute before compilation and after execution
   List queryHooks;
 
+  // a lock is used for synchronizing the state transition and its associated
+  // resource releases
+  private final ReentrantLock stateLock = new ReentrantLock();
+  private DriverState driverState = DriverState.INITIALIZED;
+
+  private enum DriverState {
+INITIALIZED,
+COMPILING,
+COMPILED,
+EXECUTING,
+EXECUTED,
+// a state that the driver enters after close() has been called to 
interrupt its running
+// query in the query cancellation
+INTERRUPT,
+// a state that the driver enters after close() has been called to clean 
the query results
+// and release the resources after the query has been executed
+CLOSED,
+// a state that the driver enters after destroy() is called and it is the 
end of driver life cycle
+DESTROYED,
+ERROR
+  }
+
   private boolean checkConcurrency() {
 boolean supportConcurrency = 
conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY);
 if (!supportConcurrency) {
@@ -350,9 +370,22 @@ public class Driver implements CommandProcessor {
* @return 0 for ok
*/
   public int compile(String command, boolean resetTaskIds) {
+return compile(command, resetTaskIds, false);
+  }
+
+  // deferClose indicates if the close/destroy should be deferred when the 
process has been
+  // interrupted, it should be set to true if the compile is called within 
another method like
+  // runInternal, which defers the close to the called in that method.
+  public int compile(String command, boolean resetTaskIds, boolean deferClose) 
{
 PerfLogger perfLogger = SessionState.getPerfLogger(true);
 perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_RUN);
 perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.COMPILE);
+stateLock.lock();
+try {
+  driverState = DriverState.COMPILING;
+} finally {
+  stateLock.unlock();
+}
 
 command = new VariableSubstitution(new HiveVariableSource() {
   @Override
@@ -370,8 +403,13 @@ public class Driver implements CommandProcessor {
   LOG.warn("WARNING! Query command could not be redacted." + e);
 }
 
+if (isInterrupted()) {
+  return handleInterruption("at beginning of compilation."); //indicate if 
need clean resource
+}
+
 if (ctx != null && ctx.getExplainAnalyze() != AnalyzeState.RUNNING) {
-  close();
+  // close the existing ctx etc before compiling a new query, but does not 
destroy driver
+  closeInProcess(false);
 }
 
 if (resetTaskIds) {
@@ -411,9 +449,13 @@ public class Driver implements CommandProcessor {
   };
   ShutdownHookManager.addShutdownHook(shutdownRunner, 
SHUTDOWN_HOOK_PRIORITY);
 
+  if (isInterrupted()) {
+return handleInterruption("before parsing and analysing the query");
+  }
   if (ctx == null) {
 ctx = new Context(conf);
   }
+
   ctx.setTryCount(getTryCount());
   ctx.setCmd(command);
   ctx.setHDFSCleanup(true);
@@ -477,9 +519,12 @@ public class Driver implements CommandProcessor {
   acidInQuery = sem.hasAcidInQuery();
   perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ANALYZE);
 
+  if (isInterrupted()) {
+

hive git commit: HIVE-14966: JDBC: Make cookie-auth work in HTTP mode (Gopal V reviewed by Tao Li, Vaibhav Gumashta)

2016-10-15 Thread vgumashta 
Repository: hive
Updated Branches:
  refs/heads/master 16d28b343 -> c71ef4fed


HIVE-14966: JDBC: Make cookie-auth work in HTTP mode (Gopal V reviewed by Tao 
Li, Vaibhav Gumashta)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c71ef4fe
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c71ef4fe
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c71ef4fe

Branch: refs/heads/master
Commit: c71ef4fed771cdd2373ca693a417d716618bf0ec
Parents: 16d28b3
Author: Vaibhav Gumashta 
Authored: Sat Oct 15 00:45:47 2016 -0700
Committer: Vaibhav Gumashta 
Committed: Sat Oct 15 00:45:47 2016 -0700

--
 common/src/java/org/apache/hadoop/hive/conf/HiveConf.java| 3 ++-
 .../java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java  | 1 -
 .../org/apache/hive/service/cli/thrift/ThriftHttpServlet.java| 4 ++--
 .../hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java  | 1 -
 4 files changed, 4 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/c71ef4fe/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 18b98e9..8ffae3b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2302,8 +2302,9 @@ public class HiveConf extends Configuration {
 "Domain for the HS2 generated cookies"),
 
HIVE_SERVER2_THRIFT_HTTP_COOKIE_PATH("hive.server2.thrift.http.cookie.path", 
null,
 "Path for the HS2 generated cookies"),
+@Deprecated
 
HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_SECURE("hive.server2.thrift.http.cookie.is.secure",
 true,
-"Secure attribute of the HS2 generated cookie."),
+"Deprecated: Secure attribute of the HS2 generated cookie (this is 
automatically enabled for SSL enabled HiveServer2)."),
 
HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_HTTPONLY("hive.server2.thrift.http.cookie.is.httponly",
 true,
 "HttpOnly attribute of the HS2 generated cookie."),
 

http://git-wip-us.apache.org/repos/asf/hive/blob/c71ef4fe/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java
--
diff --git 
a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java
 
b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java
index 98438ed..5e70d68 100644
--- 
a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java
+++ 
b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java
@@ -55,7 +55,6 @@ public class TestJdbcWithMiniKdcCookie {
 // set a small time unit as cookie max age so that the server sends a 401
 hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE,
   1, TimeUnit.SECONDS);
-hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_SECURE, 
false);
 hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
 miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf);
 miniHS2 = MiniHiveKdc.getMiniHS2WithKerb(miniHiveKdc, hiveConf);

http://git-wip-us.apache.org/repos/asf/hive/blob/c71ef4fe/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
--
diff --git 
a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java 
b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
index 50449e0..fbe6da4 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java
@@ -114,8 +114,8 @@ public class ThriftHttpServlet extends TServlet {
 ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE, TimeUnit.SECONDS);
   this.cookieDomain = 
hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_DOMAIN);
   this.cookiePath = 
hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_PATH);
-  this.isCookieSecure = hiveConf.getBoolVar(
-ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_SECURE);
+  // always send secure cookies for SSL mode
+  this.isCookieSecure = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL);
   this.isHttpOnlyCookie = hiveConf.getBoolVar(
 ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_HTTPONLY);
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/c71ef4fe/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java