(incubator-gluten) branch main updated: [GLUTEN-5001][CORE] Support mapping to native function for HiveGenericUDF (#5002)

[philo]

This is an automated email from the ASF dual-hosted git repository.

philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 2780799db [GLUTEN-5001][CORE] Support mapping to native function for 
HiveGenericUDF (#5002)
2780799db is described below

commit 2780799db5e37470c5ca6a4f21c336491ce4f808
Author: WangGuangxin <wangguangxin...@bytedance.com>
AuthorDate: Wed Mar 20 14:09:53 2024 +0800

    [GLUTEN-5001][CORE] Support mapping to native function for HiveGenericUDF 
(#5002)
---
 .../expression/ExpressionConverter.scala           |  6 ++--
 ...FTransformer.scala => HiveUDFTransformer.scala} | 32 ++++++++++++----------
 2 files changed, 21 insertions(+), 17 deletions(-)

diff --git 
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
 
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
index 278f40dd8..f407d14d6 100644
--- 
a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
+++ 
b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala
@@ -31,7 +31,7 @@ import 
org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero
 import org.apache.spark.sql.execution.{ScalarSubquery, _}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
 import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec
-import org.apache.spark.sql.hive.HiveSimpleUDFTransformer
+import org.apache.spark.sql.hive.HiveUDFTransformer
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
@@ -112,8 +112,8 @@ object ExpressionConverter extends SQLConfHelper with 
Logging {
         return replacePythonUDFWithExpressionTransformer(p, attributeSeq, 
expressionsMap)
       case s: ScalaUDF =>
         return replaceScalaUDFWithExpressionTransformer(s, attributeSeq, 
expressionsMap)
-      case _ if HiveSimpleUDFTransformer.isHiveSimpleUDF(expr) =>
-        return HiveSimpleUDFTransformer.replaceWithExpressionTransformer(expr, 
attributeSeq)
+      case _ if HiveUDFTransformer.isHiveUDF(expr) =>
+        return HiveUDFTransformer.replaceWithExpressionTransformer(expr, 
attributeSeq)
       case _ =>
     }
 
diff --git 
a/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveSimpleUDFTransformer.scala
 b/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveUDFTransformer.scala
similarity index 61%
rename from 
gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveSimpleUDFTransformer.scala
rename to 
gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveUDFTransformer.scala
index 1672648b3..5363ad139 100644
--- 
a/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveSimpleUDFTransformer.scala
+++ 
b/gluten-core/src/main/scala/org/apache/spark/sql/hive/HiveUDFTransformer.scala
@@ -16,16 +16,17 @@
  */
 package org.apache.spark.sql.hive
 
+import io.glutenproject.exception.GlutenNotSupportException
 import io.glutenproject.expression.{ExpressionConverter, 
ExpressionTransformer, GenericExpressionTransformer, UDFMappings}
 
 import org.apache.spark.sql.catalyst.expressions._
 
 import java.util.Locale
 
-object HiveSimpleUDFTransformer {
-  def isHiveSimpleUDF(expr: Expression): Boolean = {
+object HiveUDFTransformer {
+  def isHiveUDF(expr: Expression): Boolean = {
     expr match {
-      case _: HiveSimpleUDF => true
+      case _: HiveSimpleUDF | _: HiveGenericUDF => true
       case _ => false
     }
   }
@@ -33,23 +34,26 @@ object HiveSimpleUDFTransformer {
   def replaceWithExpressionTransformer(
       expr: Expression,
       attributeSeq: Seq[Attribute]): ExpressionTransformer = {
-    if (!isHiveSimpleUDF(expr)) {
-      throw new UnsupportedOperationException(s"Expression $expr is not a 
HiveSimpleUDF")
+    val udfName = expr match {
+      case s: HiveSimpleUDF =>
+        s.name.stripPrefix("default.")
+      case g: HiveGenericUDF =>
+        g.name.stripPrefix("default.")
+      case _ =>
+        throw new GlutenNotSupportException(
+          s"Expression $expr is not a HiveSimpleUDF or HiveGenericUDF")
     }
 
-    val udf = expr.asInstanceOf[HiveSimpleUDF]
-    val substraitExprName =
-      
UDFMappings.hiveUDFMap.get(udf.name.stripPrefix("default.").toLowerCase(Locale.ROOT))
-    substraitExprName match {
+    UDFMappings.hiveUDFMap.get(udfName.toLowerCase(Locale.ROOT)) match {
       case Some(name) =>
         GenericExpressionTransformer(
           name,
-          ExpressionConverter.replaceWithExpressionTransformer(udf.children, 
attributeSeq),
-          udf)
+          ExpressionConverter.replaceWithExpressionTransformer(expr.children, 
attributeSeq),
+          expr)
       case _ =>
-        throw new UnsupportedOperationException(
-          s"Not supported hive simple udf:$udf"
-            + s" name:${udf.name} hiveUDFMap:${UDFMappings.hiveUDFMap}")
+        throw new GlutenNotSupportException(
+          s"Not supported hive udf:$expr"
+            + s" name:$udfName hiveUDFMap:${UDFMappings.hiveUDFMap}")
     }
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@gluten.apache.org
For additional commands, e-mail: commits-h...@gluten.apache.org