This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 14218119251 [SPARK-38173][SQL][3.2] Quoted column cannot be recognized
correctly when quotedRegexColumnNames is true
14218119251 is described below
commit 142181192517a84cfa4569f26011eb14f68a604d
Author: huaxingao <[email protected]>
AuthorDate: Tue Jan 10 12:59:32 2023 -0800
[SPARK-38173][SQL][3.2] Quoted column cannot be recognized correctly when
quotedRegexColumnNames is true
### What changes were proposed in this pull request?
backporting https://github.com/apache/spark/pull/35476 to 3.2
### Why are the changes needed?
bug fixing in 3.2
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
new UT
Closes #39473 from huaxingao/3.2.
Authored-by: huaxingao <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 14 ++++++++++++--
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 13 +++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index d3623aead51..4db52976cb0 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2030,6 +2030,14 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with
SQLConfHelper with Logg
return false
}
+ /**
+ * Returns whether the pattern is a regex expression (instead of a normal
+ * string). Normal string is a string with all alphabets/digits and "_".
+ */
+ private def isRegex(pattern: String): Boolean = {
+ pattern.exists(p => !Character.isLetterOrDigit(p) && p != '_')
+ }
+
/**
* Create a dereference expression. The return type depends on the type of
the parent.
* If the parent is an [[UnresolvedAttribute]], it can be a
[[UnresolvedAttribute]] or
@@ -2042,7 +2050,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with
SQLConfHelper with Logg
case unresolved_attr @ UnresolvedAttribute(nameParts) =>
ctx.fieldName.getStart.getText match {
case escapedIdentifier(columnNameRegex)
- if conf.supportQuotedRegexColumnName && canApplyRegex(ctx) =>
+ if conf.supportQuotedRegexColumnName &&
+ isRegex(columnNameRegex) && canApplyRegex(ctx) =>
UnresolvedRegex(columnNameRegex, Some(unresolved_attr.name),
conf.caseSensitiveAnalysis)
case _ =>
@@ -2060,7 +2069,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with
SQLConfHelper with Logg
override def visitColumnReference(ctx: ColumnReferenceContext): Expression =
withOrigin(ctx) {
ctx.getStart.getText match {
case escapedIdentifier(columnNameRegex)
- if conf.supportQuotedRegexColumnName && canApplyRegex(ctx) =>
+ if conf.supportQuotedRegexColumnName &&
+ isRegex(columnNameRegex) && canApplyRegex(ctx) =>
UnresolvedRegex(columnNameRegex, None, conf.caseSensitiveAnalysis)
case _ =>
UnresolvedAttribute.quoted(ctx.getText)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 13b47a65dc3..6d63fa258e3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -4259,6 +4259,19 @@ class SQLQuerySuite extends QueryTest with
SharedSparkSession with AdaptiveSpark
sql("select * from test_temp_view"),
Row(1, 2, 3, 1, 2, 3, 1, 1))
}
+
+ test("SPARK-38173: Quoted column cannot be recognized correctly " +
+ "when quotedRegexColumnNames is true") {
+ withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "true") {
+ checkAnswer(
+ sql(
+ """
+ |SELECT `(C3)?+.+`,T.`C1` * `C2` AS CC
+ |FROM (SELECT 3 AS C1,2 AS C2,1 AS C3) T
+ |""".stripMargin),
+ Row(3, 2, 6) :: Nil)
+ }
+ }
}
case class Foo(bar: Option[String])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
