cloud-fan commented on code in PR #46077:
URL: https://github.com/apache/spark/pull/46077#discussion_r1577451945
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala:
##########
@@ -543,25 +555,29 @@ case class RLike(left: Expression, right: Expression)
extends StringRegexExpress
case class StringSplit(str: Expression, regex: Expression, limit: Expression)
extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
- override def dataType: DataType = ArrayType(StringType, containsNull = false)
- override def inputTypes: Seq[DataType] = Seq(StringType, StringType,
IntegerType)
+ override def dataType: DataType = ArrayType(str.dataType, containsNull =
false)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeBinaryLcase, StringTypeAnyCollation, IntegerType)
override def first: Expression = str
override def second: Expression = regex
override def third: Expression = limit
+ final lazy val collationId: Int =
str.dataType.asInstanceOf[StringType].collationId
+
def this(exp: Expression, regex: Expression) = this(exp, regex, Literal(-1))
override def nullSafeEval(string: Any, regex: Any, limit: Any): Any = {
- val strings = string.asInstanceOf[UTF8String].split(
- regex.asInstanceOf[UTF8String], limit.asInstanceOf[Int])
+ val pattern =
CollationSupport.collationAwareRegex(regex.asInstanceOf[UTF8String],
collationId)
+ val strings = string.asInstanceOf[UTF8String].split(pattern,
limit.asInstanceOf[Int])
new GenericArrayData(strings.asInstanceOf[Array[Any]])
}
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val arrayClass = classOf[GenericArrayData].getName
nullSafeCodeGen(ctx, ev, (str, regex, limit) => {
// Array in java is covariant, so we don't need to cast UTF8String[] to
Object[].
- s"""${ev.value} = new
$arrayClass($str.split($regex,$limit));""".stripMargin
+ s"""${ev.value} = new $arrayClass($str.split(
Review Comment:
this doesn't seem to match the interpreted version.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala:
##########
@@ -543,25 +555,29 @@ case class RLike(left: Expression, right: Expression)
extends StringRegexExpress
case class StringSplit(str: Expression, regex: Expression, limit: Expression)
extends TernaryExpression with ImplicitCastInputTypes with NullIntolerant {
- override def dataType: DataType = ArrayType(StringType, containsNull = false)
- override def inputTypes: Seq[DataType] = Seq(StringType, StringType,
IntegerType)
+ override def dataType: DataType = ArrayType(str.dataType, containsNull =
false)
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(StringTypeBinaryLcase, StringTypeAnyCollation, IntegerType)
override def first: Expression = str
override def second: Expression = regex
override def third: Expression = limit
+ final lazy val collationId: Int =
str.dataType.asInstanceOf[StringType].collationId
+
def this(exp: Expression, regex: Expression) = this(exp, regex, Literal(-1))
override def nullSafeEval(string: Any, regex: Any, limit: Any): Any = {
- val strings = string.asInstanceOf[UTF8String].split(
- regex.asInstanceOf[UTF8String], limit.asInstanceOf[Int])
+ val pattern =
CollationSupport.collationAwareRegex(regex.asInstanceOf[UTF8String],
collationId)
+ val strings = string.asInstanceOf[UTF8String].split(pattern,
limit.asInstanceOf[Int])
new GenericArrayData(strings.asInstanceOf[Array[Any]])
}
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val arrayClass = classOf[GenericArrayData].getName
nullSafeCodeGen(ctx, ev, (str, regex, limit) => {
// Array in java is covariant, so we don't need to cast UTF8String[] to
Object[].
- s"""${ev.value} = new
$arrayClass($str.split($regex,$limit));""".stripMargin
+ s"""${ev.value} = new $arrayClass($str.split(
Review Comment:
this doesn't seem to match the interpreted version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
