spark git commit: [SPARK-11188] [SQL] Elide stacktraces in bin/spark-sql for AnalysisExceptions

2015-11-05 Thread marmbrus 
Repository: spark
Updated Branches:
  refs/heads/branch-1.4 6c5e9a3a0 -> 4f98014b9


[SPARK-11188] [SQL] Elide stacktraces in bin/spark-sql for AnalysisExceptions

Only print the error message to the console for Analysis Exceptions in sql-shell

Author: Dilip Biswal 

Closes #9375 from dilipbiswal/spark-11188-v142.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4f98014b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4f98014b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4f98014b

Branch: refs/heads/branch-1.4
Commit: 4f98014b9b2ad66e356dbbbfdc8df4fb0f497dd9
Parents: 6c5e9a3
Author: Dilip Biswal 
Authored: Thu Nov 5 16:52:22 2015 -0800
Committer: Michael Armbrust 
Committed: Thu Nov 5 16:52:22 2015 -0800

--
 .../thriftserver/AbstractSparkSQLDriver.scala   | 23 ---
 .../hive/thriftserver/SparkSQLCLIDriver.scala   | 24 +---
 .../spark/sql/hive/thriftserver/CliSuite.scala  | 30 +++-
 3 files changed, 62 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/4f98014b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/AbstractSparkSQLDriver.scala
--
diff --git 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/AbstractSparkSQLDriver.scala
 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/AbstractSparkSQLDriver.scala
index 48ac906..df025d0 100644
--- 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/AbstractSparkSQLDriver.scala
+++ 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/AbstractSparkSQLDriver.scala
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.Driver
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse
 
 import org.apache.spark.Logging
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.hive.{HiveContext, HiveMetastoreTypes}
 
 private[hive] abstract class AbstractSparkSQLDriver(
@@ -58,13 +59,23 @@ private[hive] abstract class AbstractSparkSQLDriver(
   hiveResponse = execution.stringResult()
   tableSchema = getResultSetSchema(execution)
   new CommandProcessorResponse(0)
-} catch {
-  case cause: Throwable =>
-logError(s"Failed in [$command]", cause)
-new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(cause), 
null)
 }
   }
 
+  def runWrapper(command: String): CommandProcessorResponseWrapper = try {
+val result = run(command)
+new CommandProcessorResponseWrapper(result, null)
+  } catch {
+case ae: AnalysisException =>
+  logDebug(s"Failed in [$command]", ae)
+  new CommandProcessorResponseWrapper(new CommandProcessorResponse(1,
+ExceptionUtils.getStackTrace(ae), null), ae)
+case cause: Throwable =>
+  logError(s"Failed in [$command]", cause)
+  new CommandProcessorResponseWrapper(new CommandProcessorResponse(1,
+ExceptionUtils.getStackTrace(cause), null), cause)
+  }
+
   override def close(): Int = {
 hiveResponse = null
 tableSchema = null
@@ -79,3 +90,7 @@ private[hive] abstract class AbstractSparkSQLDriver(
 tableSchema = null
   }
 }
+
+private[hive] case class CommandProcessorResponseWrapper(
+rc : CommandProcessorResponse,
+cause : Throwable)

http://git-wip-us.apache.org/repos/asf/spark/blob/4f98014b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
--
diff --git 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index c56d807..3928f7b 100644
--- 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -17,13 +17,12 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-import scala.collection.JavaConversions._
-
 import java.io._
 import java.util.{ArrayList => JArrayList}
 
-import jline.{ConsoleReader, History}
+import scala.collection.JavaConversions._
 
+import jline.{ConsoleReader, History}
 import org.apache.commons.lang3.StringUtils
 import org.apache.commons.logging.LogFactory
 import org.apache.hadoop.conf.Configuration
@@ -32,13 +31,14 @@ import 
org.apache.hadoop.hive.common.{HiveInterruptCallback, HiveInterruptUtils}
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.ql.Driver
 import org.apache.hadoop.hive.ql.exec.Utilities
-import org.apache.hadoop.hive.ql.processor

spark git commit: [SPARK-11188] [SQL] Elide stacktraces in bin/spark-sql for AnalysisExceptions

2015-11-03 Thread marmbrus 
Repository: spark
Updated Branches:
  refs/heads/branch-1.5 b85bf8f49 -> 5604ce9c1


[SPARK-11188] [SQL] Elide stacktraces in bin/spark-sql for AnalysisExceptions

Only print the error message to the console for Analysis Exceptions in sql-shell

Author: Dilip Biswal 

Closes #9374 from dilipbiswal/dkb-11188-v152 and squashes the following commits:

a58cedc [Dilip Biswal] [SPARK-11188][SQL] Elide stacktraces in bin/spark-sql 
for AnalysisExceptions


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5604ce9c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5604ce9c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5604ce9c

Branch: refs/heads/branch-1.5
Commit: 5604ce9c1b9bfb3b8ad046915eeea787289d281e
Parents: b85bf8f
Author: Dilip Biswal 
Authored: Tue Nov 3 12:14:01 2015 +0100
Committer: Michael Armbrust 
Committed: Tue Nov 3 12:14:01 2015 +0100

--
 .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 10 +-
 .../spark/sql/hive/thriftserver/SparkSQLDriver.scala| 10 +++---
 .../apache/spark/sql/hive/thriftserver/CliSuite.scala   | 12 ++--
 3 files changed, 26 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/5604ce9c/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
--
diff --git 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index e58f8ca..212bd2c 100644
--- 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -22,6 +22,8 @@ import scala.collection.JavaConversions._
 import java.io._
 import java.util.{ArrayList => JArrayList, Locale}
 
+import org.apache.spark.sql.AnalysisException
+
 import jline.console.ConsoleReader
 import jline.console.history.FileHistory
 
@@ -298,6 +300,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver 
with Logging {
 
   driver.init()
   val out = sessionState.out
+  val err = sessionState.err
   val start: Long = System.currentTimeMillis()
   if (sessionState.getIsVerbose) {
 out.println(cmd)
@@ -308,7 +311,12 @@ private[hive] class SparkSQLCLIDriver extends CliDriver 
with Logging {
 
   ret = rc.getResponseCode
   if (ret != 0) {
-console.printError(rc.getErrorMessage())
+// For analysis exception, only the error is printed out to the 
console.
+rc.getException() match {
+  case e : AnalysisException =>
+err.println(s"""Error in query: ${e.getMessage}""")
+  case _ => err.println(rc.getErrorMessage())
+}
 driver.close()
 return ret
   }

http://git-wip-us.apache.org/repos/asf/spark/blob/5604ce9c/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
--
diff --git 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
index 77272ae..e44fa5e 100644
--- 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
+++ 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.hive.thriftserver
 
 import java.util.{ArrayList => JArrayList, List => JList}
+import org.apache.spark.sql.AnalysisException
 
 import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.hadoop.hive.metastore.api.{FieldSchema, Schema}
@@ -63,9 +64,12 @@ private[hive] class SparkSQLDriver(
   tableSchema = getResultSetSchema(execution)
   new CommandProcessorResponse(0)
 } catch {
-  case cause: Throwable =>
-logError(s"Failed in [$command]", cause)
-new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(cause), 
null)
+case ae: AnalysisException =>
+  logDebug(s"Failed in [$command]", ae)
+  new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(ae), 
null, ae)
+case cause: Throwable =>
+  logError(s"Failed in [$command]", cause)
+  new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(cause), 
null, cause)
 }
   }
 

http://git-wip-us.apache.org/repos/asf/spark/blob/5604ce9c/sql/hive-thriftserver/src/test/scala

spark git commit: [SPARK-11188][SQL] Elide stacktraces in bin/spark-sql for AnalysisExceptions

2015-10-29 Thread marmbrus 
Repository: spark
Updated Branches:
  refs/heads/master f7a51deeb -> 8185f038c


[SPARK-11188][SQL] Elide stacktraces in bin/spark-sql for AnalysisExceptions

Only print the error message to the console for Analysis Exceptions in 
sql-shell.

Author: Dilip Biswal 

Closes #9194 from dilipbiswal/spark-11188.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8185f038
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8185f038
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8185f038

Branch: refs/heads/master
Commit: 8185f038c13c72e1bea7b0921b84125b7a352139
Parents: f7a51de
Author: Dilip Biswal 
Authored: Thu Oct 29 18:29:50 2015 +0100
Committer: Michael Armbrust 
Committed: Thu Oct 29 18:29:50 2015 +0100

--
 .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 10 +-
 .../spark/sql/hive/thriftserver/SparkSQLDriver.scala| 11 ---
 .../apache/spark/sql/hive/thriftserver/CliSuite.scala   | 12 ++--
 3 files changed, 27 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/8185f038/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
--
diff --git 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index b507396..62e912c 100644
--- 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.hive.thriftserver
 import java.io._
 import java.util.{ArrayList => JArrayList, Locale}
 
+import org.apache.spark.sql.AnalysisException
+
 import scala.collection.JavaConverters._
 
 import jline.console.ConsoleReader
@@ -298,6 +300,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver 
with Logging {
 
   driver.init()
   val out = sessionState.out
+  val err = sessionState.err
   val start: Long = System.currentTimeMillis()
   if (sessionState.getIsVerbose) {
 out.println(cmd)
@@ -308,7 +311,12 @@ private[hive] class SparkSQLCLIDriver extends CliDriver 
with Logging {
 
   ret = rc.getResponseCode
   if (ret != 0) {
-console.printError(rc.getErrorMessage())
+// For analysis exception, only the error is printed out to the 
console.
+rc.getException() match {
+  case e : AnalysisException =>
+err.println(s"""Error in query: ${e.getMessage}""")
+  case _ => err.println(rc.getErrorMessage())
+}
 driver.close()
 return ret
   }

http://git-wip-us.apache.org/repos/asf/spark/blob/8185f038/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
--
diff --git 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
index 2619286..f1ec723 100644
--- 
a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
+++ 
b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
@@ -18,6 +18,8 @@
 package org.apache.spark.sql.hive.thriftserver
 
 import java.util.{Arrays, ArrayList => JArrayList, List => JList}
+import org.apache.log4j.LogManager
+import org.apache.spark.sql.AnalysisException
 
 import scala.collection.JavaConverters._
 
@@ -63,9 +65,12 @@ private[hive] class SparkSQLDriver(
   tableSchema = getResultSetSchema(execution)
   new CommandProcessorResponse(0)
 } catch {
-  case cause: Throwable =>
-logError(s"Failed in [$command]", cause)
-new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(cause), 
null)
+case ae: AnalysisException =>
+  logDebug(s"Failed in [$command]", ae)
+  new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(ae), 
null, ae)
+case cause: Throwable =>
+  logError(s"Failed in [$command]", cause)
+  new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(cause), 
null, cause)
 }
   }
 

http://git-wip-us.apache.org/repos/asf/spark/blob/8185f038/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
--
diff --git 
a/sql/hive-thriftserver/src/test/scala