jinchengchenghh commented on code in PR #9107:
URL: https://github.com/apache/incubator-gluten/pull/9107#discussion_r2424889874
##########
backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala:
##########
@@ -622,6 +622,47 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ // Add test suite for CharVarcharCodegenUtils functions.
+ // A ProjectExecTransformer is expected to be constructed after expr support.
+ // We currently test below functions with Spark v3.4
+ testWithMinSparkVersion("charTypeWriteSideCheck", "3.4") {
Review Comment:
Why only test with Spark3,4? Does Spark 3.3 not have this function?
##########
gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala:
##########
@@ -144,29 +144,89 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
DecimalArithmeticExpressionTransformer(substraitName, leftChild,
rightChild, resultType, b)
}
- private def replaceIcebergStaticInvoke(
- s: StaticInvoke,
+ private def replaceStaticInvokeWithExpressionTransformer(
+ i: StaticInvoke,
attributeSeq: Seq[Attribute],
expressionsMap: Map[Class[_], String]): ExpressionTransformer = {
- val invokeMap = Map(
- "BucketFunction" -> ExpressionNames.BUCKET,
- "TruncateFunction" -> ExpressionNames.TRUNCATE,
- "YearsFunction" -> ExpressionNames.YEARS,
- "MonthsFunction" -> ExpressionNames.MONTHS,
- "DaysFunction" -> ExpressionNames.DAYS,
- "HoursFunction" -> ExpressionNames.HOURS
- )
- val objName = s.staticObject.getName
- val transformer = invokeMap.find {
- case (func, _) =>
objName.startsWith("org.apache.iceberg.spark.functions." + func)
+ def validateAndTransform(
+ exprName: String,
+ childTransformers: => Seq[ExpressionTransformer]):
ExpressionTransformer = {
+ if (!BackendsApiManager.getValidatorApiInstance.doExprValidate(exprName,
i)) {
+ throw new GlutenNotSupportException(
+ s"Not supported to map current ${i.getClass} call on function:
${i.functionName}.")
+ }
+ GenericExpressionTransformer(exprName, childTransformers, i)
+ }
+
+ if (Seq("encode", "decode").contains(i.functionName) &&
i.objectName.endsWith("UrlCodec")) {
+ return validateAndTransform(
+ "url_" + i.functionName,
+ Seq(replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap))
+ )
+ }
+
+ if (i.functionName.equals("isLuhnNumber")) {
+ return validateAndTransform(
+ ExpressionNames.LUHN_CHECK,
+ Seq(replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap))
+ )
+ }
+
+ if (Seq("encode", "decode").contains(i.functionName) &&
i.objectName.endsWith("Base64")) {
+ // Respect backend-specific implementation while still validating the
expr name
+ if
(!BackendsApiManager.getValidatorApiInstance.doExprValidate(ExpressionNames.BASE64,
i)) {
+ throw new GlutenNotSupportException(
+ s"Not supported to map current ${i.getClass} call on function:
${i.functionName}.")
+ }
+ return
BackendsApiManager.getSparkPlanExecApiInstance.genBase64StaticInvokeTransformer(
+ ExpressionNames.BASE64,
+ replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap),
+ i
+ )
+ }
+
+ if (
+ Set("varcharTypeWriteSideCheck", "charTypeWriteSideCheck",
"readSidePadding").contains(
+ i.functionName) && i.objectName.endsWith("CharVarcharCodegenUtils")
+ ) {
+ val exprName = i.functionName match {
+ case "varcharTypeWriteSideCheck" =>
ExpressionNames.VARCHAR_TYPE_WRITE_SIDE_CHECK
+ case "charTypeWriteSideCheck" =>
ExpressionNames.CHAR_TYPE_WRITE_SIDE_CHECK
+ case "readSidePadding" => ExpressionNames.READ_SIDE_PADDING
+ }
+ return validateAndTransform(
+ exprName,
+ i.arguments.map(replaceWithExpressionTransformer0(_, attributeSeq,
expressionsMap))
+ )
}
- if (transformer.isEmpty) {
- throw new GlutenNotSupportException(s"Not supported staticInvoke call
object: $objName")
+
+ if (
Review Comment:
Please do not expand the replaceIcebergStaticInvoke here, this function is a
bit long and will be longger
##########
gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala:
##########
@@ -144,29 +144,89 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
DecimalArithmeticExpressionTransformer(substraitName, leftChild,
rightChild, resultType, b)
}
- private def replaceIcebergStaticInvoke(
- s: StaticInvoke,
+ private def replaceStaticInvokeWithExpressionTransformer(
+ i: StaticInvoke,
attributeSeq: Seq[Attribute],
expressionsMap: Map[Class[_], String]): ExpressionTransformer = {
- val invokeMap = Map(
- "BucketFunction" -> ExpressionNames.BUCKET,
- "TruncateFunction" -> ExpressionNames.TRUNCATE,
- "YearsFunction" -> ExpressionNames.YEARS,
- "MonthsFunction" -> ExpressionNames.MONTHS,
- "DaysFunction" -> ExpressionNames.DAYS,
- "HoursFunction" -> ExpressionNames.HOURS
- )
- val objName = s.staticObject.getName
- val transformer = invokeMap.find {
- case (func, _) =>
objName.startsWith("org.apache.iceberg.spark.functions." + func)
+ def validateAndTransform(
+ exprName: String,
+ childTransformers: => Seq[ExpressionTransformer]):
ExpressionTransformer = {
+ if (!BackendsApiManager.getValidatorApiInstance.doExprValidate(exprName,
i)) {
+ throw new GlutenNotSupportException(
+ s"Not supported to map current ${i.getClass} call on function:
${i.functionName}.")
+ }
+ GenericExpressionTransformer(exprName, childTransformers, i)
+ }
+
+ if (Seq("encode", "decode").contains(i.functionName) &&
i.objectName.endsWith("UrlCodec")) {
Review Comment:
Could you use case match for the function name? This will be more clean
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
