spark git commit: [SPARK-10121] [SQL] Thrift server always use the latest class loader provided by the conf of executionHive's state

2015-08-24 Thread lian 
Repository: spark
Updated Branches:
  refs/heads/branch-1.5 2f7e4b416 -> c99f4160b


[SPARK-10121] [SQL] Thrift server always use the latest class loader provided 
by the conf of executionHive's state

https://issues.apache.org/jira/browse/SPARK-10121

Looks like the problem is that if we add a jar through another thread, the 
thread handling the JDBC session will not get the latest classloader.

Author: Yin Huai 

Closes #8368 from yhuai/SPARK-10121.

(cherry picked from commit a0c0aae1defe5e1e57704065631d201f8e3f6bac)
Signed-off-by: Cheng Lian 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c99f4160
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c99f4160
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c99f4160

Branch: refs/heads/branch-1.5
Commit: c99f4160b98bc0685c23fee4eb7b892c47f6feda
Parents: 2f7e4b4
Author: Yin Huai 
Authored: Tue Aug 25 12:49:50 2015 +0800
Committer: Cheng Lian 
Committed: Tue Aug 25 12:50:44 2015 +0800

--
 .../SparkExecuteStatementOperation.scala|  6 +++
 .../thriftserver/HiveThriftServer2Suites.scala  | 54 
 2 files changed, 60 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/c99f4160/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
--
diff --git 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index 833bf62..02cc7e5 100644
--- 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -159,6 +159,12 @@ private[hive] class SparkExecuteStatementOperation(
 
   // User information is part of the metastore client member in 
Hive
   hiveContext.setSession(currentSqlSession)
+  // Always use the latest class loader provided by 
executionHive's state.
+  val executionHiveClassLoader =
+hiveContext.executionHive.state.getConf.getClassLoader
+  sessionHive.getConf.setClassLoader(executionHiveClassLoader)
+  
parentSessionState.getConf.setClassLoader(executionHiveClassLoader)
+
   Hive.set(sessionHive)
   SessionState.setCurrentSessionState(parentSessionState)
   try {

http://git-wip-us.apache.org/repos/asf/spark/blob/c99f4160/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
--
diff --git 
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
 
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index ded42bc..b72249b 100644
--- 
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ 
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -377,6 +377,60 @@ class HiveThriftBinaryServerSuite extends 
HiveThriftJdbcTest {
   rs2.close()
 }
   }
+
+  test("test add jar") {
+withMultipleConnectionJdbcStatement(
+  {
+statement =>
+  val jarFile =
+"../hive/src/test/resources/hive-hcatalog-core-0.13.1.jar"
+  .split("/")
+  .mkString(File.separator)
+
+  statement.executeQuery(s"ADD JAR $jarFile")
+  },
+
+  {
+statement =>
+  val queries = Seq(
+"DROP TABLE IF EXISTS smallKV",
+"CREATE TABLE smallKV(key INT, val STRING)",
+s"LOAD DATA LOCAL INPATH '${TestData.smallKv}' OVERWRITE INTO 
TABLE smallKV",
+"DROP TABLE IF EXISTS addJar",
+"""CREATE TABLE addJar(key string)
+  |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
+""".stripMargin)
+
+  queries.foreach(statement.execute)
+
+  statement.executeQuery(
+"""
+  |INSERT INTO TABLE addJar SELECT 'k1' as key FROM smallKV limit 1
+""".stripMargin)
+
+  val actualResult =
+statement.executeQuery("SELECT key FROM addJar")
+  val actualResultBuffer = new collection.mutable.ArrayBuffer[String]()
+  while (actualResult.next()) {
+actualResultBuffer += actualResult.getString(1)
+  }
+  actualResult.close()
+
+  val exp

spark git commit: [SPARK-10121] [SQL] Thrift server always use the latest class loader provided by the conf of executionHive's state

2015-08-24 Thread lian 
Repository: spark
Updated Branches:
  refs/heads/master 642c43c81 -> a0c0aae1d


[SPARK-10121] [SQL] Thrift server always use the latest class loader provided 
by the conf of executionHive's state

https://issues.apache.org/jira/browse/SPARK-10121

Looks like the problem is that if we add a jar through another thread, the 
thread handling the JDBC session will not get the latest classloader.

Author: Yin Huai 

Closes #8368 from yhuai/SPARK-10121.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a0c0aae1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a0c0aae1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a0c0aae1

Branch: refs/heads/master
Commit: a0c0aae1defe5e1e57704065631d201f8e3f6bac
Parents: 642c43c
Author: Yin Huai 
Authored: Tue Aug 25 12:49:50 2015 +0800
Committer: Cheng Lian 
Committed: Tue Aug 25 12:49:50 2015 +0800

--
 .../SparkExecuteStatementOperation.scala|  6 +++
 .../thriftserver/HiveThriftServer2Suites.scala  | 54 
 2 files changed, 60 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/a0c0aae1/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
--
diff --git 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index 833bf62..02cc7e5 100644
--- 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -159,6 +159,12 @@ private[hive] class SparkExecuteStatementOperation(
 
   // User information is part of the metastore client member in 
Hive
   hiveContext.setSession(currentSqlSession)
+  // Always use the latest class loader provided by 
executionHive's state.
+  val executionHiveClassLoader =
+hiveContext.executionHive.state.getConf.getClassLoader
+  sessionHive.getConf.setClassLoader(executionHiveClassLoader)
+  
parentSessionState.getConf.setClassLoader(executionHiveClassLoader)
+
   Hive.set(sessionHive)
   SessionState.setCurrentSessionState(parentSessionState)
   try {

http://git-wip-us.apache.org/repos/asf/spark/blob/a0c0aae1/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
--
diff --git 
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
 
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index ded42bc..b72249b 100644
--- 
a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ 
b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -377,6 +377,60 @@ class HiveThriftBinaryServerSuite extends 
HiveThriftJdbcTest {
   rs2.close()
 }
   }
+
+  test("test add jar") {
+withMultipleConnectionJdbcStatement(
+  {
+statement =>
+  val jarFile =
+"../hive/src/test/resources/hive-hcatalog-core-0.13.1.jar"
+  .split("/")
+  .mkString(File.separator)
+
+  statement.executeQuery(s"ADD JAR $jarFile")
+  },
+
+  {
+statement =>
+  val queries = Seq(
+"DROP TABLE IF EXISTS smallKV",
+"CREATE TABLE smallKV(key INT, val STRING)",
+s"LOAD DATA LOCAL INPATH '${TestData.smallKv}' OVERWRITE INTO 
TABLE smallKV",
+"DROP TABLE IF EXISTS addJar",
+"""CREATE TABLE addJar(key string)
+  |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
+""".stripMargin)
+
+  queries.foreach(statement.execute)
+
+  statement.executeQuery(
+"""
+  |INSERT INTO TABLE addJar SELECT 'k1' as key FROM smallKV limit 1
+""".stripMargin)
+
+  val actualResult =
+statement.executeQuery("SELECT key FROM addJar")
+  val actualResultBuffer = new collection.mutable.ArrayBuffer[String]()
+  while (actualResult.next()) {
+actualResultBuffer += actualResult.getString(1)
+  }
+  actualResult.close()
+
+  val expectedResult =
+statement.executeQuery("SELECT 'k1'")
+  val expectedResultBuffer = ne