Github user andrewor14 commented on a diff in the pull request:
https://github.com/apache/spark/pull/12036#discussion_r57944268
--- Diff: sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
---
@@ -55,61 +57,133 @@ private[hive] class HiveFunctionRegistry(
}
}
- override def lookupFunction(name: String, children: Seq[Expression]):
Expression = {
- Try(underlying.lookupFunction(name, children)).getOrElse {
- // We only look it up to see if it exists, but do not include it in
the HiveUDF since it is
- // not always serializable.
- val functionInfo: FunctionInfo =
- Option(getFunctionInfo(name.toLowerCase)).getOrElse(
- throw new AnalysisException(s"undefined function $name"))
+ def loadHivePermanentFunction(name: String): Option[CatalogFunction] = {
+ val databaseName = sessionStage.catalog.getCurrentDatabase
+ val func = FunctionIdentifier(name, Option(databaseName))
+ val catalogFunc =
+ if (sessionStage.catalog.listFunctions(databaseName, name).size !=
0) {
+ Some(sessionStage.catalog.getFunction(func))
+ } else {
+ None
+ }
+ catalogFunc.map(_.resources.foreach { resource =>
+ resource._1.toLowerCase match {
+ case "jar" => sessionStage.ctx.addJar(resource._2)
+ case _ =>
+ sessionStage.ctx.runSqlHive(s"ADD FILE ${resource._2}")
+ sessionStage.ctx.sparkContext.addFile(resource._2)
+ }
+ })
+ catalogFunc
+ }
- val functionClassName = functionInfo.getFunctionClass.getName
+ override def makeFunctionBuilderAndInfo(
+ name: String,
+ functionClassName: String): (ExpressionInfo, FunctionBuilder) = {
+ val hiveUDFWrapper = new HiveFunctionWrapper(functionClassName)
+ val hiveUDFClass = hiveUDFWrapper.createFunction().getClass
+ val info = new ExpressionInfo(functionClassName, name)
--- End diff --
This is the whole reason why you have `makeFunctionBuilderAndInfo`. The
signature here is clunky and unnecessary. We can just pass `functionClassName`
to the `SessionCatalog` and do that there ourselves when we create a temporary
function. Then we don't need all these tuple return types and this can be just
`makeFunctionBuilder`.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
