Repository: spark
Updated Branches:
refs/heads/branch-1.2 0e7fa7f63 -> 97b7eb4d9
[SPARK-4487][SQL] Fix attribute reference resolution error when using ORDER BY.
When we use ORDER BY clause, at first, attributes referenced by projection are
resolved (1).
And then, attributes referenced at ORDER BY clause are resolved (2).
But when resolving attributes referenced at ORDER BY clause, the resolution
result generated in (1) is discarded so for example, following query fails.
SELECT c1 + c2 FROM mytable ORDER BY c1;
The query above fails because when resolving the attribute reference 'c1', the
resolution result of 'c2' is discarded.
Author: Kousuke Saruta <[email protected]>
Closes #3363 from sarutak/SPARK-4487 and squashes the following commits:
fd314f3 [Kousuke Saruta] Fixed attribute resolution logic in Analyzer
6e60c20 [Kousuke Saruta] Fixed conflicts
cb5b7e9 [Kousuke Saruta] Added test case for SPARK-4487
282d529 [Kousuke Saruta] Fixed attributes reference resolution error
b6123e6 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark
into concat-feature
317b7fb [Kousuke Saruta] WIP
(cherry picked from commit dd1c9cb36cde8202cede8014b5641ae8a0197812)
Signed-off-by: Michael Armbrust <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/97b7eb4d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/97b7eb4d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/97b7eb4d
Branch: refs/heads/branch-1.2
Commit: 97b7eb4d99613944d39f1421dccc2724c4165c9e
Parents: 0e7fa7f
Author: Kousuke Saruta <[email protected]>
Authored: Mon Nov 24 12:54:37 2014 -0800
Committer: Michael Armbrust <[email protected]>
Committed: Mon Nov 24 12:54:50 2014 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 +-
.../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 7 +++++++
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/97b7eb4d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index d3b4cf8..facbd8b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -179,7 +179,7 @@ class Analyzer(catalog: Catalog, registry:
FunctionRegistry, caseSensitive: Bool
val missingInProject = requiredAttributes -- p.output
if (missingInProject.nonEmpty) {
// Add missing attributes and then project them away after the sort.
- Project(projectList,
+ Project(projectList.map(_.toAttribute),
Sort(ordering,
Project(projectList ++ missingInProject, child)))
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/97b7eb4d/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 0a96831..84ee305 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -974,6 +974,13 @@ class SQLQuerySuite extends QueryTest with
BeforeAndAfterAll {
dropTempTable("data")
}
+ test("SPARK-4432 Fix attribute reference resolution error when using ORDER
BY") {
+ checkAnswer(
+ sql("SELECT a + b FROM testData2 ORDER BY a"),
+ Seq(2, 3, 3 ,4 ,4 ,5).map(Seq(_))
+ )
+ }
+
test("Supporting relational operator '<=>' in Spark SQL") {
val nullCheckData1 = TestData(1,"1") :: TestData(2,null) :: Nil
val rdd1 = sparkContext.parallelize((0 to 1).map(i => nullCheckData1(i)))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
