This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1ae1d89d08a [SPARK-43205][SQL][FOLLOWUP] IDENTIFIER clause should
accept alias and RuntimeReplaceable
1ae1d89d08a is described below
commit 1ae1d89d08a7c3997c65099ebfdb9b5955483bc5
Author: Wenchen Fan <wenc...@databricks.com>
AuthorDate: Tue Aug 15 23:47:28 2023 +0800
[SPARK-43205][SQL][FOLLOWUP] IDENTIFIER clause should accept alias and
RuntimeReplaceable
### What changes were proposed in this pull request?
This is a followup of https://github.com/apache/spark/pull/41007
Similar to `ResolveInlineTables`, we need to trim aliases and replace
`RuntimeReplaceable` expressions before evaluating an expression in
`ResolveIdentifierClause`.
### Why are the changes needed?
Make IDENTIFIER support more cases
### Does this PR introduce _any_ user-facing change?
Yes, but IDENTIFIER is not released yet.
### How was this patch tested?
new tests
Closes #42497 from cloud-fan/variable.
Authored-by: Wenchen Fan <wenc...@databricks.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../analysis/ResolveIdentifierClause.scala | 6 ++--
.../catalyst/analysis/ResolveInlineTables.scala | 15 +++++-----
.../sql/catalyst/expressions/EvalHelper.scala | 32 ++++++++++++++++++++++
.../sql/catalyst/analysis/AnalysisSuite.scala | 17 ++++++++++++
4 files changed, 59 insertions(+), 11 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveIdentifierClause.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveIdentifierClause.scala
index 2a1e17d390e..e0d3e5629ef 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveIdentifierClause.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveIdentifierClause.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.catalyst.analysis
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.{AliasHelper, EvalHelper,
Expression}
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.rules.Rule
@@ -27,7 +27,7 @@ import org.apache.spark.sql.types.StringType
/**
* Resolves the identifier expressions and builds the original
plans/expressions.
*/
-object ResolveIdentifierClause extends Rule[LogicalPlan] {
+object ResolveIdentifierClause extends Rule[LogicalPlan] with AliasHelper with
EvalHelper {
override def apply(plan: LogicalPlan): LogicalPlan =
plan.resolveOperatorsUpWithPruning(
_.containsAnyPattern(UNRESOLVED_IDENTIFIER)) {
@@ -41,7 +41,7 @@ object ResolveIdentifierClause extends Rule[LogicalPlan] {
}
private def evalIdentifierExpr(expr: Expression): Seq[String] = {
- expr match {
+ trimAliases(prepareForEval(expr)) match {
case e if !e.foldable => expr.failAnalysis(
errorClass = "NOT_A_CONSTANT_STRING.NOT_CONSTANT",
messageParameters = Map(
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
index c203f9fa39d..760ea466b85 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInlineTables.scala
@@ -20,8 +20,7 @@ package org.apache.spark.sql.catalyst.analysis
import scala.util.control.NonFatal
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.AliasHelper
-import org.apache.spark.sql.catalyst.optimizer.ReplaceExpressions
+import org.apache.spark.sql.catalyst.expressions.{AliasHelper, EvalHelper}
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.trees.AlwaysProcess
@@ -32,14 +31,14 @@ import org.apache.spark.sql.types.{StructField, StructType}
/**
* An analyzer rule that replaces [[UnresolvedInlineTable]] with
[[LocalRelation]].
*/
-object ResolveInlineTables extends Rule[LogicalPlan] with CastSupport with
AliasHelper {
+object ResolveInlineTables extends Rule[LogicalPlan]
+ with CastSupport with AliasHelper with EvalHelper {
override def apply(plan: LogicalPlan): LogicalPlan =
plan.resolveOperatorsWithPruning(
AlwaysProcess.fn, ruleId) {
case table: UnresolvedInlineTable if table.expressionsResolved =>
validateInputDimension(table)
- val newTable =
ReplaceExpressions(table).asInstanceOf[UnresolvedInlineTable]
- validateInputEvaluable(newTable)
- convert(newTable)
+ validateInputEvaluable(table)
+ convert(table)
}
/**
@@ -75,7 +74,7 @@ object ResolveInlineTables extends Rule[LogicalPlan] with
CastSupport with Alias
table.rows.foreach { row =>
row.foreach { e =>
// Note that nondeterministic expressions are not supported since they
are not foldable.
- if (!e.resolved || !trimAliases(e).foldable) {
+ if (!e.resolved || !trimAliases(prepareForEval(e)).foldable) {
e.failAnalysis(
errorClass =
"INVALID_INLINE_TABLE.CANNOT_EVALUATE_EXPRESSION_IN_INLINE_TABLE",
messageParameters = Map("expr" -> toSQLExpr(e)))
@@ -115,7 +114,7 @@ object ResolveInlineTables extends Rule[LogicalPlan] with
CastSupport with Alias
} else {
cast(e, targetType)
}
- castedExpr.eval()
+ prepareForEval(castedExpr).eval()
} catch {
case NonFatal(ex) =>
table.failAnalysis(
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EvalHelper.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EvalHelper.scala
new file mode 100644
index 00000000000..2818ec3b9bf
--- /dev/null
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EvalHelper.scala
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.catalyst.trees.TreePattern.RUNTIME_REPLACEABLE
+
+/**
+ * Helper methods for evaluating expressions.
+ */
+trait EvalHelper {
+
+ def prepareForEval(e: Expression): Expression = {
+ e.transformWithPruning(_.containsAnyPattern(RUNTIME_REPLACEABLE)) {
+ case r: RuntimeReplaceable => r.replacement
+ }
+ }
+}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 539b32e7b3b..f8fedc0500c 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -1658,4 +1658,21 @@ class AnalysisSuite extends AnalysisTest with Matchers {
plan.join(plan).analyze
}
}
+
+ test("IDENTIFIER with alias and RuntimeReplaceable") {
+ val name = Literal("a").as("name")
+ val replaceable = new Nvl(Literal("a"), Literal("b"))
+ withClue("IDENTIFIER as column") {
+ val ident = ExpressionWithUnresolvedIdentifier(name,
UnresolvedAttribute.apply)
+ checkAnalysis(testRelation.select(ident),
testRelation.select($"a").analyze)
+ val ident2 = ExpressionWithUnresolvedIdentifier(replaceable,
UnresolvedAttribute.apply)
+ checkAnalysis(testRelation.select(ident2),
testRelation.select($"a").analyze)
+ }
+ withClue("IDENTIFIER as table") {
+ val ident = PlanWithUnresolvedIdentifier(name, _ => testRelation)
+ checkAnalysis(ident.select($"a"), testRelation.select($"a").analyze)
+ val ident2 = PlanWithUnresolvedIdentifier(replaceable, _ => testRelation)
+ checkAnalysis(ident2.select($"a"), testRelation.select($"a").analyze)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
