rui-mo commented on code in PR #9107:
URL: https://github.com/apache/incubator-gluten/pull/9107#discussion_r2301382095
##########
gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala:
##########
@@ -144,6 +144,67 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
DecimalArithmeticExpressionTransformer(substraitName, leftChild,
rightChild, resultType, b)
}
+ private def replaceStaticInvokeWithExpressionTransformer(
+ i: StaticInvoke,
+ attributeSeq: Seq[Attribute],
+ expressionsMap: Map[Class[_], String]): ExpressionTransformer = {
+ def validateAndTransform(
+ exprName: String,
+ childTransformers: => Seq[ExpressionTransformer]):
ExpressionTransformer = {
+ if (!BackendsApiManager.getValidatorApiInstance.doExprValidate(exprName,
i)) {
+ throw new GlutenNotSupportException(
+ s"Not supported to map current ${i.getClass} call on function:
${i.functionName}.")
+ }
+ GenericExpressionTransformer(exprName, childTransformers, i)
+ }
+
+ if (Seq("encode", "decode").contains(i.functionName) &&
i.objectName.endsWith("UrlCodec")) {
+ return validateAndTransform(
+ "url_" + i.functionName,
+ Seq(replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap))
+ )
+ }
+
+ if (i.functionName == "isLuhnNumber") {
+ return validateAndTransform(
+ ExpressionNames.LUHN_CHECK,
+ Seq(replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap))
+ )
+ }
+
+ if (Seq("encode", "decode").contains(i.functionName) &&
i.objectName.endsWith("Base64")) {
+ // Respect backend-specific implementation while still validating the
expr name
+ if
(!BackendsApiManager.getValidatorApiInstance.doExprValidate(ExpressionNames.BASE64,
i)) {
+ throw new GlutenNotSupportException(
+ s"Not supported to map current ${i.getClass} call on function:
${i.functionName}.")
+ }
+ return
BackendsApiManager.getSparkPlanExecApiInstance.genBase64StaticInvokeTransformer(
+ ExpressionNames.BASE64,
+ replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap),
+ i
+ )
+ }
+
+ if (
+ Set("varcharTypeWriteSideCheck", "charTypeWriteSideCheck",
"readSidePadding").contains(
Review Comment:
ditto
##########
gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala:
##########
@@ -144,6 +144,67 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
DecimalArithmeticExpressionTransformer(substraitName, leftChild,
rightChild, resultType, b)
}
+ private def replaceStaticInvokeWithExpressionTransformer(
+ i: StaticInvoke,
+ attributeSeq: Seq[Attribute],
+ expressionsMap: Map[Class[_], String]): ExpressionTransformer = {
+ def validateAndTransform(
+ exprName: String,
+ childTransformers: => Seq[ExpressionTransformer]):
ExpressionTransformer = {
+ if (!BackendsApiManager.getValidatorApiInstance.doExprValidate(exprName,
i)) {
+ throw new GlutenNotSupportException(
+ s"Not supported to map current ${i.getClass} call on function:
${i.functionName}.")
+ }
+ GenericExpressionTransformer(exprName, childTransformers, i)
+ }
+
+ if (Seq("encode", "decode").contains(i.functionName) &&
i.objectName.endsWith("UrlCodec")) {
+ return validateAndTransform(
+ "url_" + i.functionName,
+ Seq(replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap))
+ )
+ }
+
+ if (i.functionName == "isLuhnNumber") {
Review Comment:
ditto
##########
gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala:
##########
@@ -144,6 +144,67 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
DecimalArithmeticExpressionTransformer(substraitName, leftChild,
rightChild, resultType, b)
}
+ private def replaceStaticInvokeWithExpressionTransformer(
+ i: StaticInvoke,
+ attributeSeq: Seq[Attribute],
+ expressionsMap: Map[Class[_], String]): ExpressionTransformer = {
+ def validateAndTransform(
+ exprName: String,
+ childTransformers: => Seq[ExpressionTransformer]):
ExpressionTransformer = {
+ if (!BackendsApiManager.getValidatorApiInstance.doExprValidate(exprName,
i)) {
+ throw new GlutenNotSupportException(
+ s"Not supported to map current ${i.getClass} call on function:
${i.functionName}.")
+ }
+ GenericExpressionTransformer(exprName, childTransformers, i)
+ }
+
+ if (Seq("encode", "decode").contains(i.functionName) &&
i.objectName.endsWith("UrlCodec")) {
Review Comment:
Are the comparisons case-sensitive?
##########
gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala:
##########
@@ -144,6 +144,67 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
DecimalArithmeticExpressionTransformer(substraitName, leftChild,
rightChild, resultType, b)
}
+ private def replaceStaticInvokeWithExpressionTransformer(
+ i: StaticInvoke,
+ attributeSeq: Seq[Attribute],
+ expressionsMap: Map[Class[_], String]): ExpressionTransformer = {
+ def validateAndTransform(
+ exprName: String,
+ childTransformers: => Seq[ExpressionTransformer]):
ExpressionTransformer = {
+ if (!BackendsApiManager.getValidatorApiInstance.doExprValidate(exprName,
i)) {
+ throw new GlutenNotSupportException(
+ s"Not supported to map current ${i.getClass} call on function:
${i.functionName}.")
+ }
+ GenericExpressionTransformer(exprName, childTransformers, i)
+ }
+
+ if (Seq("encode", "decode").contains(i.functionName) &&
i.objectName.endsWith("UrlCodec")) {
+ return validateAndTransform(
+ "url_" + i.functionName,
+ Seq(replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap))
+ )
+ }
+
+ if (i.functionName == "isLuhnNumber") {
+ return validateAndTransform(
+ ExpressionNames.LUHN_CHECK,
+ Seq(replaceWithExpressionTransformer0(i.arguments.head, attributeSeq,
expressionsMap))
+ )
+ }
+
+ if (Seq("encode", "decode").contains(i.functionName) &&
i.objectName.endsWith("Base64")) {
Review Comment:
ditto
##########
backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala:
##########
@@ -622,6 +622,47 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ // Add test suite for CharVarcharCodegenUtils functions.
+ // A ProjectExecTransformer is expected to be constructed after expr support.
+ // We currently test below functions with Spark v3.4
+ testWithMinSparkVersion("Test charTypeWriteSideCheck function", "3.4") {
Review Comment:
nit: could be rename as "charTypeWriteSideCheck" to be consistent in style
with other cases. And are these functions only supported after Spark 3.4?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
