HeartSaVioR edited a comment on issue #27025: [SPARK-26560][SQL] Spark should be able to run Hive UDF using jar regardless of current thread context classloader URL: https://github.com/apache/spark/pull/27025#issuecomment-569495909 ``` 17:55:15.408 ERROR org.apache.spark.SparkContext: Failed to add <a href='file:///home/jenkins/workspace/SparkPullRequestBuilder/sql/hive/src/test/noclasspath/TestUDTF.jar'>file:///home/jenkins/workspace/SparkPullRequestBuilder/sql/hive/src/test/noclasspath/TestUDTF.jar</a> to Spark environment java.lang.IllegalArgumentException: requirement failed: File TestUDTF.jar was already registered with a different path (old path = /home/jenkins/workspace/SparkPullRequestBuilder/sql/hive/target/scala-2.12/test-classes/TestUDTF.jar, new path = /home/jenkins/workspace/SparkPullRequestBuilder/sql/hive/src/test/noclasspath/TestUDTF.jar at scala.Predef$.require(Predef.scala:281) at org.apache.spark.rpc.netty.NettyStreamManager.addJar(NettyStreamManager.scala:79) at org.apache.spark.SparkContext.addLocalJarFile$1(SparkContext.scala:1853) at org.apache.spark.SparkContext.addJar(SparkContext.scala:1901) at org.apache.spark.sql.internal.SessionResourceLoader.addJar(SessionState.scala:166) at org.apache.spark.sql.hive.HiveSessionResourceLoader.addJar(HiveSessionStateBuilder.scala:118) at org.apache.spark.sql.internal.SessionResourceLoader.loadResource(SessionState.scala:149) at org.apache.spark.sql.catalyst.catalog.SessionCatalog.$anonfun$loadFunctionResources$1(SessionCatalog.scala:1287) at org.apache.spark.sql.catalyst.catalog.SessionCatalog.$anonfun$loadFunctionResources$1$adapted(SessionCatalog.scala:1287) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.sql.catalyst.catalog.SessionCatalog.loadFunctionResources(SessionCatalog.scala:1287) at org.apache.spark.sql.catalyst.catalog.SessionCatalog.lookupFunction(SessionCatalog.scala:1428) at org.apache.spark.sql.hive.HiveSessionCatalog.super$lookupFunction(HiveSessionCatalog.scala:135) at org.apache.spark.sql.hive.HiveSessionCatalog.$anonfun$lookupFunction0$2(HiveSessionCatalog.scala:135) at scala.util.Try$.apply(Try.scala:213) at org.apache.spark.sql.hive.HiveSessionCatalog.lookupFunction0(HiveSessionCatalog.scala:135) at org.apache.spark.sql.hive.HiveSessionCatalog.lookupFunction(HiveSessionCatalog.scala:121) at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$15$$anonfun$applyOrElse$96.$anonfun$applyOrElse$99(Analyzer.scala:1739) at org.apache.spark.sql.catalyst.analysis.package$.withPosition(package.scala:53) at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$15$$anonfun$applyOrElse$96.applyOrElse(Analyzer.scala:1739) at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$15$$anonfun$applyOrElse$96.applyOrElse(Analyzer.scala:1722) at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$1(TreeNode.scala:286) at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:72) at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:286) at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:291) at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:376) ... ``` It doesn't seem to allow same name - I'll just rename the jar to `TestUDTF-spark-26560.jar` after the jar name.
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
