This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c0943b349de7 [SPARK-48885][SQL] Make some subclasses of
RuntimeReplaceable override replacement to lazy val
c0943b349de7 is described below
commit c0943b349de7e38e2a36ca144b903e4bd11fc042
Author: Kent Yao <[email protected]>
AuthorDate: Tue Jul 16 10:18:12 2024 +0800
[SPARK-48885][SQL] Make some subclasses of RuntimeReplaceable override
replacement to lazy val
### What changes were proposed in this pull request?
This PR makes 8 subclasses of RuntimeReplaceable override replacement to
lazy val to align with other 60+ members and avoid recreation of new expressions
```scala
Value read (51 usages found)
spark-catalyst_2.13 (50 usages found)
AnyValue.scala (1 usage found)
54 override lazy val replacement: Expression =
First(child, ignoreNulls)
arithmetic.scala (1 usage found)
127 override lazy val replacement: Expression = child
bitmapExpressions.scala (3 usages found)
52 override lazy val replacement: Expression =
StaticInvoke(
85 override lazy val replacement: Expression =
StaticInvoke(
134 override lazy val replacement: Expression =
StaticInvoke(
boolAggregates.scala (2 usages found)
39 override lazy val replacement: Expression =
Min(child)
61 override lazy val replacement: Expression =
Max(child)
collationExpressions.scala (1 usage found)
123 override def replacement: Expression = {
collectionOperations.scala (5 usages found)
168 override lazy val replacement: Expression =
Size(child, legacySizeOfNull = false)
231 override lazy val replacement: Expression =
ArrayContains(MapKeys(left), right)
1596 override lazy val replacement: Expression = new
ArrayInsert(left, Literal(1), right)
1631 override lazy val replacement: Expression = new
ArrayInsert(left, Literal(-1), right)
5203 override lazy val replacement: Expression =
ArrayFilter(child, lambda)
CountIf.scala (1 usage found)
42 override lazy val replacement: Expression =
Count(new NullIf(child, Literal.FalseLiteral))
datetimeExpressions.scala (2 usages found)
2070 override lazy val replacement: Expression =
format.map { f =>
2145 override lazy val replacement: Expression =
format.map { f =>
linearRegression.scala (5 usages found)
45 override lazy val replacement: Expression =
Count(Seq(left, right))
79 override lazy val replacement: Expression =
114 override lazy val replacement: Expression =
176 override lazy val replacement: Expression =
232 override lazy val replacement: Expression =
misc.scala (3 usages found)
294 override lazy val replacement: Expression =
StaticInvoke(
397 override lazy val replacement: Expression =
StaticInvoke(
475 override lazy val replacement: Expression =
StaticInvoke(
percentiles.scala (2 usages found)
346 override def replacement: Expression = percentile
365 override def replacement: Expression = percentile
regexpExpressions.scala (3 usages found)
262 override lazy val replacement: Expression =
Like(Lower(left), Lower(right), escapeChar)
1034 override lazy val replacement: Expression =
1072 override lazy val replacement: Expression =
stringExpressions.scala (14 usages found)
561 override lazy val replacement =
723 override lazy val replacement: Expression =
Invoke(input, "isValid", BooleanType)
770 override lazy val replacement: Expression =
Invoke(input, "makeValid", input.dataType)
810 override lazy val replacement: Expression =
StaticInvoke(
859 override lazy val replacement: Expression =
StaticInvoke(
1854 override lazy val replacement: Expression =
StaticInvoke(
2246 override lazy val replacement: Expression = If(
2284 override lazy val replacement: Expression =
Substring(str, Literal(1), len)
2713 override def replacement: Expression =
StaticInvoke(
2940 override def replacement: Expression =
StaticInvoke(
3004 override val replacement: Expression =
StaticInvoke(
3075 override lazy val replacement: Expression = if
(fmt == null) {
3473 override lazy val replacement: Expression =
3533 override lazy val replacement: Expression =
StaticInvoke(
toFromAvroSqlFunctions.scala (2 usages found)
96 override def replacement: Expression = {
168 override def replacement: Expression = {
urlExpressions.scala (2 usages found)
55 override def replacement: Expression =
92 override def replacement: Expression =
variantExpressions.scala (3 usages found)
58 override lazy val replacement: Expression =
StaticInvoke(
100 override lazy val replacement: Expression =
StaticInvoke(
635 override lazy val replacement: Expression =
StaticInvoke(
spark-examples_2.13 (1 usage found)
AgeExample.scala (1 usage found)
27 override lazy val replacement: Expression =
SubtractDates(CurrentDate(), birthday)
```
### Why are the changes needed?
Improve RuntimeReplaceable implementations
### Does this PR introduce _any_ user-facing change?
NO
### How was this patch tested?
existing tests
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #47333 from yaooqinn/SPARK-48885.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../spark/sql/catalyst/expressions/aggregate/percentiles.scala | 4 ++--
.../spark/sql/catalyst/expressions/collationExpressions.scala | 2 +-
.../spark/sql/catalyst/expressions/stringExpressions.scala | 10 +++++-----
.../sql/catalyst/expressions/toFromAvroSqlFunctions.scala | 4 ++--
.../apache/spark/sql/catalyst/expressions/urlExpressions.scala | 4 ++--
5 files changed, 12 insertions(+), 12 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala
index c074b19f2419..77d2bc5b4228 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala
@@ -343,7 +343,7 @@ case class Median(child: Expression)
with ImplicitCastInputTypes
with UnaryLike[Expression] {
private lazy val percentile = new Percentile(child, Literal(0.5, DoubleType))
- override def replacement: Expression = percentile
+ override lazy val replacement: Expression = percentile
override def nodeName: String = "median"
override def inputTypes: Seq[AbstractDataType] =
percentile.inputTypes.take(1)
override protected def withNewChildInternal(
@@ -362,7 +362,7 @@ case class PercentileCont(left: Expression, right:
Expression, reverse: Boolean
with SupportsOrderingWithinGroup
with BinaryLike[Expression] {
private lazy val percentile = new Percentile(left, right, reverse)
- override def replacement: Expression = percentile
+ override lazy val replacement: Expression = percentile
override def nodeName: String = "percentile_cont"
override def inputTypes: Seq[AbstractDataType] = percentile.inputTypes
override def sql(isDistinct: Boolean): String = {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala
index c7fbb39ea285..7a428e8ed45a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collationExpressions.scala
@@ -120,7 +120,7 @@ case class Collate(child: Expression, collationName: String)
case class Collation(child: Expression)
extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes {
override protected def withNewChildInternal(newChild: Expression): Collation
= copy(newChild)
- override def replacement: Expression = {
+ override lazy val replacement: Expression = {
val collationId = child.dataType.asInstanceOf[StringType].collationId
val collationName =
CollationFactory.fetchCollation(collationId).collationName
Literal.create(collationName, SQLConf.get.defaultStringType)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index fec782002bb7..08f373a86ae3 100755
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -2707,10 +2707,10 @@ case class Base64(child: Expression, chunkBase64:
Boolean)
def this(expr: Expression) = this(expr, SQLConf.get.chunkBase64StringEnabled)
- override def dataType: DataType = SQLConf.get.defaultStringType
+ override val dataType: DataType = SQLConf.get.defaultStringType
override def inputTypes: Seq[DataType] = Seq(BinaryType)
- override def replacement: Expression = StaticInvoke(
+ override lazy val replacement: Expression = StaticInvoke(
classOf[Base64],
dataType,
"encode",
@@ -2932,12 +2932,12 @@ case class StringDecode(
def this(bin: Expression, charset: Expression) =
this(bin, charset, SQLConf.get.legacyJavaCharsets,
SQLConf.get.legacyCodingErrorAction)
- override def dataType: DataType = SQLConf.get.defaultStringType
+ override val dataType: DataType = SQLConf.get.defaultStringType
override def inputTypes: Seq[AbstractDataType] = Seq(BinaryType,
StringTypeAnyCollation)
override def prettyName: String = "decode"
override def toString: String = s"$prettyName($bin, $charset)"
- override def replacement: Expression = StaticInvoke(
+ override lazy val replacement: Expression = StaticInvoke(
classOf[StringDecode],
SQLConf.get.defaultStringType,
"decode",
@@ -3001,7 +3001,7 @@ case class Encode(
override def inputTypes: Seq[AbstractDataType] =
Seq(StringTypeAnyCollation, StringTypeAnyCollation)
- override val replacement: Expression = StaticInvoke(
+ override lazy val replacement: Expression = StaticInvoke(
classOf[Encode],
BinaryType,
"encode",
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromAvroSqlFunctions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromAvroSqlFunctions.scala
index ca53058230fb..88a53e459118 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromAvroSqlFunctions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/toFromAvroSqlFunctions.scala
@@ -93,7 +93,7 @@ case class FromAvro(child: Expression, jsonFormatSchema:
Expression, options: Ex
TypeCheckResult.TypeCheckSuccess))
}
- override def replacement: Expression = {
+ override lazy val replacement: Expression = {
val schemaValue: String = jsonFormatSchema.eval() match {
case s: UTF8String =>
s.toString
@@ -165,7 +165,7 @@ case class ToAvro(child: Expression, jsonFormatSchema:
Expression)
}
}
- override def replacement: Expression = {
+ override lazy val replacement: Expression = {
val schemaValue: Option[String] = jsonFormatSchema.eval() match {
case null =>
None
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala
index 65fce955b8f8..1687b69e46be 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/urlExpressions.scala
@@ -52,7 +52,7 @@ import org.apache.spark.unsafe.types.UTF8String
case class UrlEncode(child: Expression)
extends RuntimeReplaceable with UnaryLike[Expression] with
ImplicitCastInputTypes {
- override def replacement: Expression =
+ override lazy val replacement: Expression =
StaticInvoke(
UrlCodec.getClass,
SQLConf.get.defaultStringType,
@@ -89,7 +89,7 @@ case class UrlEncode(child: Expression)
case class UrlDecode(child: Expression)
extends RuntimeReplaceable with UnaryLike[Expression] with
ImplicitCastInputTypes {
- override def replacement: Expression =
+ override lazy val replacement: Expression =
StaticInvoke(
UrlCodec.getClass,
SQLConf.get.defaultStringType,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
