This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1ef5638 [SPARK-38173][SQL] Quoted column cannot be recognized
correctly when quotedRegexColumnNa…
1ef5638 is described below
commit 1ef5638177dcf06ebca4e9b0bc88401e0fce2ae8
Author: TongWeii <[email protected]>
AuthorDate: Wed Feb 16 12:40:59 2022 +0800
[SPARK-38173][SQL] Quoted column cannot be recognized correctly when
quotedRegexColumnNa…
### What changes were proposed in this pull request?
bug fix
### Why are the changes needed?
When spark.sql.parser.quotedRegexColumnNames=true
```
SELECT `(C3)?+.+`,`C1` * C2 FROM (SELECT 3 AS C1,2 AS C2,1 AS C3) T;
```
The above query will throw an exception
```
Error: org.apache.hive.service.cli.HiveSQLException: Error running query:
org.apache.spark.sql.AnalysisException: Invalid usage of '*' in expression
'multiply'
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.org$apache$spark$sql$hive$thriftserver$SparkExecuteStatementOperation$$execute(SparkExecuteStatementOperation.scala:370)
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.$anonfun$run$2(SparkExecuteStatementOperation.scala:266)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at
org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties(SparkOperation.scala:78)
at
org.apache.spark.sql.hive.thriftserver.SparkOperation.withLocalProperties$(SparkOperation.scala:62)
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation.withLocalProperties(SparkExecuteStatementOperation.scala:44)
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:266)
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2$$anon$3.run(SparkExecuteStatementOperation.scala:261)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729)
at
org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperation$$anon$2.run(SparkExecuteStatementOperation.scala:275)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.spark.sql.AnalysisException: Invalid usage of '*' in
expression 'multiply'
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis.failAnalysis(CheckAnalysis.scala:50)
at
org.apache.spark.sql.catalyst.analysis.CheckAnalysis.failAnalysis$(CheckAnalysis.scala:49)
at
org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:155)
at
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$expandStarExpression$1.applyOrElse(Analyzer.scala:1700)
at
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$expandStarExpression$1.applyOrElse(Analyzer.scala:1671)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUp$2(TreeNode.scala:342)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:74)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:342)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUp$1(TreeNode.scala:339)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:408)
at
org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:244)
at
org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:406)
at
org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:359)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:339)
at
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$.expandStarExpression(Analyzer.scala:1671)
at
org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$.$anonfun$buildExpandedProjectList$1(Analyzer.scala:1656)
```
It works fine in hive, because hive treats a pattern with all
alphabets/digits and "_" as a normal string
```
/**
* Returns whether the pattern is a regex expression (instead of a normal
* string). Normal string is a string with all alphabets/digits and "_".
*/
static boolean isRegex(String pattern, HiveConf conf) {
String qIdSupport = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_QUOTEDID_SUPPORT);
if ( "column".equals(qIdSupport)) {
return false;
}
for (int i = 0; i < pattern.length(); i++) {
if (!Character.isLetterOrDigit(pattern.charAt(i))
&& pattern.charAt(i) != '_') {
return true;
}
}
return false;
}
```
```
0: jdbc:hive2://hiveserver-inc.> set hive.support.quoted.identifiers=none;
No rows affected (0.003 seconds)
0: jdbc:hive2://hiveserver-inc.> SELECT `(C3)?+.+`,`C1` * C2 FROM (SELECT 3
AS C1,2 AS C2,1 AS C3) T;
22/02/10 19:01:43 INFO ql.Driver: OK
+-------+-------+------+
| t.c1 | t.c2 | _c1 |
+-------+-------+------+
| 3 | 2 | 6 |
+-------+-------+------+
1 row selected (0.136 seconds)
```
In this pr, we add the `isRegex` method to check whether the pattern is a
regex expression
### Does this PR introduce _any_ user-facing change?
NO
### How was this patch tested?
UT
Closes #35476 from TongWei1105/SPARK-38173.
Lead-authored-by: TongWeii <[email protected]>
Co-authored-by: TongWei <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 14 ++++++++++++--
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 13 +++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index bd43cff..773d32d 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2092,6 +2092,14 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with
SQLConfHelper with Logg
}
/**
+ * Returns whether the pattern is a regex expression (instead of a normal
+ * string). Normal string is a string with all alphabets/digits and "_".
+ */
+ private def isRegex(pattern: String): Boolean = {
+ pattern.exists(p => !Character.isLetterOrDigit(p) && p != '_')
+ }
+
+ /**
* Create a dereference expression. The return type depends on the type of
the parent.
* If the parent is an [[UnresolvedAttribute]], it can be a
[[UnresolvedAttribute]] or
* a [[UnresolvedRegex]] for regex quoted in ``; if the parent is some other
expression,
@@ -2103,7 +2111,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with
SQLConfHelper with Logg
case unresolved_attr @ UnresolvedAttribute(nameParts) =>
ctx.fieldName.getStart.getText match {
case escapedIdentifier(columnNameRegex)
- if conf.supportQuotedRegexColumnName && canApplyRegex(ctx) =>
+ if conf.supportQuotedRegexColumnName &&
+ isRegex(columnNameRegex) && canApplyRegex(ctx) =>
UnresolvedRegex(columnNameRegex, Some(unresolved_attr.name),
conf.caseSensitiveAnalysis)
case _ =>
@@ -2121,7 +2130,8 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with
SQLConfHelper with Logg
override def visitColumnReference(ctx: ColumnReferenceContext): Expression =
withOrigin(ctx) {
ctx.getStart.getText match {
case escapedIdentifier(columnNameRegex)
- if conf.supportQuotedRegexColumnName && canApplyRegex(ctx) =>
+ if conf.supportQuotedRegexColumnName &&
+ isRegex(columnNameRegex) && canApplyRegex(ctx) =>
UnresolvedRegex(columnNameRegex, None, conf.caseSensitiveAnalysis)
case _ =>
UnresolvedAttribute.quoted(ctx.getText)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index ffc3db3..974e489 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -4281,6 +4281,19 @@ class SQLQuerySuite extends QueryTest with
SharedSparkSession with AdaptiveSpark
Row(2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22) :: Nil)
}
}
+
+ test("SPARK-38173: Quoted column cannot be recognized correctly " +
+ "when quotedRegexColumnNames is true") {
+ withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "true") {
+ checkAnswer(
+ sql(
+ """
+ |SELECT `(C3)?+.+`,T.`C1` * `C2` AS CC
+ |FROM (SELECT 3 AS C1,2 AS C2,1 AS C3) T
+ |""".stripMargin),
+ Row(3, 2, 6) :: Nil)
+ }
+ }
}
case class Foo(bar: Option[String])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
