Repository: spark
Updated Branches:
refs/heads/branch-2.0 988d4dbf4 -> 1bb0aa4b0
[SPARK-15388][SQL] Fix spark sql CREATE FUNCTION with hive 1.2.1
## What changes were proposed in this pull request?
spark.sql("CREATE FUNCTION myfunc AS 'com.haizhi.bdp.udf.UDFGetGeoCode'")
throws
"org.apache.hadoop.hive.ql.metadata.HiveException:MetaException(message:NoSuchObjectException(message:Function
default.myfunc does not exist))" with hive 1.2.1.
I think it is introduced by pr #12853. Fixing it by catching Exception (not
NoSuchObjectException) and string matching.
## How was this patch tested?
added a unit test and also tested it manually
Author: wangyang <[email protected]>
Closes #13177 from wangyang1992/fixCreateFunc2.
(cherry picked from commit 784cc07d1675eb9e0a387673cf86874e1bfc10f9)
Signed-off-by: Andrew Or <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1bb0aa4b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1bb0aa4b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1bb0aa4b
Branch: refs/heads/branch-2.0
Commit: 1bb0aa4b0a5b680e535cac6305b9eac856606ef4
Parents: 988d4db
Author: wangyang <[email protected]>
Authored: Tue May 24 11:03:12 2016 -0700
Committer: Andrew Or <[email protected]>
Committed: Tue May 24 11:03:21 2016 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/hive/client/HiveShim.scala | 14 ++++++++++++--
.../apache/spark/sql/hive/client/VersionsSuite.scala | 1 +
2 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1bb0aa4b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
----------------------------------------------------------------------
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 78713c3..9df4a26 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -24,6 +24,7 @@ import java.util.{ArrayList => JArrayList, List => JList, Map
=> JMap, Set => JS
import java.util.concurrent.TimeUnit
import scala.collection.JavaConverters._
+import scala.util.control.NonFatal
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.hive.conf.HiveConf
@@ -42,7 +43,6 @@ import
org.apache.spark.sql.catalyst.analysis.NoSuchPermanentFunctionException
import org.apache.spark.sql.catalyst.catalog.{CatalogFunction,
CatalogTablePartition, FunctionResource, FunctionResourceType}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types.{IntegralType, StringType}
-import org.apache.spark.util.CausedBy
/**
@@ -480,11 +480,21 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
try {
Option(hive.getFunction(db, name)).map(fromHiveFunction)
} catch {
- case CausedBy(ex: NoSuchObjectException) if ex.getMessage.contains(name)
=>
+ case NonFatal(e) if isCausedBy(e, s"$name does not exist") =>
None
}
}
+ private def isCausedBy(e: Throwable, matchMassage: String): Boolean = {
+ if (e.getMessage.contains(matchMassage)) {
+ true
+ } else if (e.getCause != null) {
+ isCausedBy(e.getCause, matchMassage)
+ } else {
+ false
+ }
+ }
+
override def listFunctions(hive: Hive, db: String, pattern: String):
Seq[String] = {
hive.getFunctions(db, pattern).asScala
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1bb0aa4b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index d46c4e7..8ae4535 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -440,6 +440,7 @@ class VersionsSuite extends SparkFunSuite with Logging {
assert(client.getFunctionOption("default", "func2").isEmpty)
} else {
assert(client.getFunctionOption("default", "func2").isDefined)
+ assert(client.getFunctionOption("default",
"the_func_not_exists").isEmpty)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
