Repository: spark
Updated Branches:
refs/heads/master 8a6bbe095 -> 26fbca480
[SPARK-17832][SQL] TableIdentifier.quotedString creates un-parseable names when
name contains a backtick
## What changes were proposed in this pull request?
The `quotedString` method in `TableIdentifier` and `FunctionIdentifier` produce
an illegal (un-parseable) name when the name contains a backtick. For example:
```
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser._
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
val complexName = TableIdentifier("`weird`table`name", Some("`d`b`1"))
parseTableIdentifier(complexName.unquotedString) // Does not work
parseTableIdentifier(complexName.quotedString) // Does not work
parseExpression(complexName.unquotedString) // Does not work
parseExpression(complexName.quotedString) // Does not work
```
We should handle the backtick properly to make `quotedString` parseable.
## How was this patch tested?
Add new testcases in `TableIdentifierParserSuite` and `ExpressionParserSuite`.
Author: jiangxingbo <[email protected]>
Closes #15403 from jiangxb1987/backtick.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/26fbca48
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/26fbca48
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/26fbca48
Branch: refs/heads/master
Commit: 26fbca480604ba258f97b9590cfd6dda1ecd31db
Parents: 8a6bbe0
Author: jiangxingbo <[email protected]>
Authored: Sun Oct 9 21:52:46 2016 -0700
Committer: Herman van Hovell <[email protected]>
Committed: Sun Oct 9 21:52:46 2016 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/identifiers.scala | 11 +++++++++--
.../sql/catalyst/parser/ExpressionParserSuite.scala | 11 ++++++++++-
.../sql/catalyst/parser/TableIdentifierParserSuite.scala | 10 ++++++++++
3 files changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/26fbca48/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
index d7b48ce..834897b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
@@ -17,7 +17,6 @@
package org.apache.spark.sql.catalyst
-
/**
* An identifier that optionally specifies a database.
*
@@ -29,8 +28,16 @@ sealed trait IdentifierWithDatabase {
def database: Option[String]
+ /*
+ * Escapes back-ticks within the identifier name with double-back-ticks.
+ */
+ private def quoteIdentifier(name: String): String = name.replace("`", "``")
+
def quotedString: String = {
- if (database.isDefined) s"`${database.get}`.`$identifier`" else
s"`$identifier`"
+ val replacedId = quoteIdentifier(identifier)
+ val replacedDb = database.map(quoteIdentifier(_))
+
+ if (replacedDb.isDefined) s"`${replacedDb.get}`.`$replacedId`" else
s"`$replacedId`"
}
def unquotedString: String = {
http://git-wip-us.apache.org/repos/asf/spark/blob/26fbca48/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 0fb1138..17cfc81 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.sql.catalyst.parser
import java.sql.{Date, Timestamp}
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, _}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.PlanTest
@@ -535,4 +535,13 @@ class ExpressionParserSuite extends PlanTest {
// ".123BD" should not be treated as token of type BIGDECIMAL_LITERAL
assertEqual("a.123BD_column", UnresolvedAttribute("a.123BD_column"))
}
+
+ test("SPARK-17832 function identifier contains backtick") {
+ val complexName = FunctionIdentifier("`ba`r", Some("`fo`o"))
+ assertEqual(complexName.quotedString, UnresolvedAttribute("`fo`o.`ba`r"))
+ intercept(complexName.unquotedString, "mismatched input")
+ // Function identifier contains countious backticks should be treated
correctly.
+ val complexName2 = FunctionIdentifier("ba``r", Some("fo``o"))
+ assertEqual(complexName2.quotedString, UnresolvedAttribute("fo``o.ba``r"))
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/26fbca48/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index 793be89..7d46011 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -104,4 +104,14 @@ class TableIdentifierParserSuite extends SparkFunSuite {
// ".123BD" should not be treated as token of type BIGDECIMAL_LITERAL
assert(parseTableIdentifier("a.123BD_LIST") ==
TableIdentifier("123BD_LIST", Some("a")))
}
+
+ test("SPARK-17832 table identifier - contains backtick") {
+ val complexName = TableIdentifier("`weird`table`name", Some("`d`b`1"))
+ assert(complexName ===
parseTableIdentifier("```d``b``1`.```weird``table``name`"))
+ assert(complexName === parseTableIdentifier(complexName.quotedString))
+ intercept[ParseException](parseTableIdentifier(complexName.unquotedString))
+ // Table identifier contains countious backticks should be treated
correctly.
+ val complexName2 = TableIdentifier("x``y", Some("d``b"))
+ assert(complexName2 === parseTableIdentifier(complexName2.quotedString))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
