Repository: spark Updated Branches: refs/heads/branch-1.2 72f5ba1fc -> 8608ff598
[SPARK-4513][SQL] Support relational operator '<=>' in Spark SQL The relational operator '<=>' is not working in Spark SQL. Same works in Spark HiveQL Author: ravipesala <[email protected]> Closes #3387 from ravipesala/<=> and squashes the following commits: 7198e90 [ravipesala] Supporting relational operator '<=>' in Spark SQL (cherry picked from commit 98e9419784a9ad5096cfd563fa9a433786a90bd4) Signed-off-by: Michael Armbrust <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8608ff59 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8608ff59 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8608ff59 Branch: refs/heads/branch-1.2 Commit: 8608ff59881b3cfa6c4cd407ba2c0af7a78e88a9 Parents: 72f5ba1 Author: ravipesala <[email protected]> Authored: Thu Nov 20 15:34:03 2014 -0800 Committer: Michael Armbrust <[email protected]> Committed: Thu Nov 20 15:34:24 2014 -0800 ---------------------------------------------------------------------- .../org/apache/spark/sql/catalyst/SparkSQLParser.scala | 2 +- .../scala/org/apache/spark/sql/catalyst/SqlParser.scala | 1 + .../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 12 ++++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/8608ff59/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala index f5c19ee..b198ed9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SparkSQLParser.scala @@ -61,7 +61,7 @@ class SqlLexical(val keywords: Seq[String]) extends StdLexical { delimiters += ( "@", "*", "+", "-", "<", "=", "<>", "!=", "<=", ">=", ">", "/", "(", ")", - ",", ";", "%", "{", "}", ":", "[", "]", ".", "&", "|", "^", "~" + ",", ";", "%", "{", "}", ":", "[", "]", ".", "&", "|", "^", "~", "<=>" ) override lazy val token: Parser[Token] = http://git-wip-us.apache.org/repos/asf/spark/blob/8608ff59/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala index affef27..dc1d349 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala @@ -234,6 +234,7 @@ class SqlParser extends AbstractSparkSQLParser { | termExpression ~ (">=" ~> termExpression) ^^ { case e1 ~ e2 => GreaterThanOrEqual(e1, e2) } | termExpression ~ ("!=" ~> termExpression) ^^ { case e1 ~ e2 => Not(EqualTo(e1, e2)) } | termExpression ~ ("<>" ~> termExpression) ^^ { case e1 ~ e2 => Not(EqualTo(e1, e2)) } + | termExpression ~ ("<=>" ~> termExpression) ^^ { case e1 ~ e2 => EqualNullSafe(e1, e2) } | termExpression ~ NOT.? ~ (BETWEEN ~> termExpression) ~ (AND ~> termExpression) ^^ { case e ~ not ~ el ~ eu => val betweenExpr: Expression = And(GreaterThanOrEqual(e, el), LessThanOrEqual(e, eu)) http://git-wip-us.apache.org/repos/asf/spark/blob/8608ff59/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index a635154..0a96831 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -973,4 +973,16 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { checkAnswer(sql("SELECT a.b + 1 FROM data GROUP BY a.b + 1"), 2) dropTempTable("data") } + + test("Supporting relational operator '<=>' in Spark SQL") { + val nullCheckData1 = TestData(1,"1") :: TestData(2,null) :: Nil + val rdd1 = sparkContext.parallelize((0 to 1).map(i => nullCheckData1(i))) + rdd1.registerTempTable("nulldata1") + val nullCheckData2 = TestData(1,"1") :: TestData(2,null) :: Nil + val rdd2 = sparkContext.parallelize((0 to 1).map(i => nullCheckData2(i))) + rdd2.registerTempTable("nulldata2") + checkAnswer(sql("SELECT nulldata1.key FROM nulldata1 join " + + "nulldata2 on nulldata1.value <=> nulldata2.value"), + (1 to 2).map(i => Seq(i))) + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
