Repository: spark
Updated Branches:
refs/heads/branch-2.0 6d056c168 -> d27df3579
[SPARK-17832][SQL] TableIdentifier.quotedString creates un-parseable names when
name contains a backtick
## What changes were proposed in this pull request?
The `quotedString` method in `TableIdentifier` and `FunctionIdentifier` produce
an illegal (un-parseable) name when the name contains a backtick. For example:
```
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser._
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
val complexName = TableIdentifier("`weird`table`name", Some("`d`b`1"))
parseTableIdentifier(complexName.unquotedString) // Does not work
parseTableIdentifier(complexName.quotedString) // Does not work
parseExpression(complexName.unquotedString) // Does not work
parseExpression(complexName.quotedString) // Does not work
```
We should handle the backtick properly to make `quotedString` parseable.
## How was this patch tested?
Add new testcases in `TableIdentifierParserSuite` and `ExpressionParserSuite`.
Author: jiangxingbo <[email protected]>
Closes #15403 from jiangxb1987/backtick.
(cherry picked from commit 26fbca480604ba258f97b9590cfd6dda1ecd31db)
Signed-off-by: Herman van Hovell <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d27df357
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d27df357
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d27df357
Branch: refs/heads/branch-2.0
Commit: d27df35795fac0fd167e51d5ba08092a17eedfc2
Parents: 6d056c1
Author: jiangxingbo <[email protected]>
Authored: Sun Oct 9 21:52:46 2016 -0700
Committer: Herman van Hovell <[email protected]>
Committed: Sun Oct 9 21:52:58 2016 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/identifiers.scala | 11 +++++++++--
.../sql/catalyst/parser/ExpressionParserSuite.scala | 11 ++++++++++-
.../sql/catalyst/parser/TableIdentifierParserSuite.scala | 10 ++++++++++
3 files changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d27df357/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
index d7b48ce..834897b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
@@ -17,7 +17,6 @@
package org.apache.spark.sql.catalyst
-
/**
* An identifier that optionally specifies a database.
*
@@ -29,8 +28,16 @@ sealed trait IdentifierWithDatabase {
def database: Option[String]
+ /*
+ * Escapes back-ticks within the identifier name with double-back-ticks.
+ */
+ private def quoteIdentifier(name: String): String = name.replace("`", "``")
+
def quotedString: String = {
- if (database.isDefined) s"`${database.get}`.`$identifier`" else
s"`$identifier`"
+ val replacedId = quoteIdentifier(identifier)
+ val replacedDb = database.map(quoteIdentifier(_))
+
+ if (replacedDb.isDefined) s"`${replacedDb.get}`.`$replacedId`" else
s"`$replacedId`"
}
def unquotedString: String = {
http://git-wip-us.apache.org/repos/asf/spark/blob/d27df357/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index c89ddad..15ebc19 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.sql.catalyst.parser
import java.sql.{Date, Timestamp}
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, _}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.PlanTest
@@ -534,4 +534,13 @@ class ExpressionParserSuite extends PlanTest {
// ".123BD" should not be treated as token of type BIGDECIMAL_LITERAL
assertEqual("a.123BD_column", UnresolvedAttribute("a.123BD_column"))
}
+
+ test("SPARK-17832 function identifier contains backtick") {
+ val complexName = FunctionIdentifier("`ba`r", Some("`fo`o"))
+ assertEqual(complexName.quotedString, UnresolvedAttribute("`fo`o.`ba`r"))
+ intercept(complexName.unquotedString, "mismatched input")
+ // Function identifier contains countious backticks should be treated
correctly.
+ val complexName2 = FunctionIdentifier("ba``r", Some("fo``o"))
+ assertEqual(complexName2.quotedString, UnresolvedAttribute("fo``o.ba``r"))
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/d27df357/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index 4d3ad21..cb57fb6 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -96,4 +96,14 @@ class TableIdentifierParserSuite extends SparkFunSuite {
// ".123BD" should not be treated as token of type BIGDECIMAL_LITERAL
assert(parseTableIdentifier("a.123BD_LIST") ==
TableIdentifier("123BD_LIST", Some("a")))
}
+
+ test("SPARK-17832 table identifier - contains backtick") {
+ val complexName = TableIdentifier("`weird`table`name", Some("`d`b`1"))
+ assert(complexName ===
parseTableIdentifier("```d``b``1`.```weird``table``name`"))
+ assert(complexName === parseTableIdentifier(complexName.quotedString))
+ intercept[ParseException](parseTableIdentifier(complexName.unquotedString))
+ // Table identifier contains countious backticks should be treated
correctly.
+ val complexName2 = TableIdentifier("x``y", Some("d``b"))
+ assert(complexName2 === parseTableIdentifier(complexName2.quotedString))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
