Repository: spark Updated Branches: refs/heads/master e25312451 -> 6aceaf3d6
[SPARK-9295] Analysis should detect sorting on unsupported column types This patch extends CheckAnalysis to throw errors for queries that try to sort on unsupported column types, such as ArrayType. Author: Josh Rosen <[email protected]> Closes #7633 from JoshRosen/SPARK-9295 and squashes the following commits: 23b2fbf [Josh Rosen] Embed function in foreach bfe1451 [Josh Rosen] Update to allow sorting by null literals 2f1b802 [Josh Rosen] Add analysis rule to detect sorting on unsupported column types (SPARK-9295) Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6aceaf3d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6aceaf3d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6aceaf3d Branch: refs/heads/master Commit: 6aceaf3d62ee335570ddc07ccaf07e8c3776f517 Parents: e253124 Author: Josh Rosen <[email protected]> Authored: Fri Jul 24 11:34:23 2015 -0700 Committer: Josh Rosen <[email protected]> Committed: Fri Jul 24 11:34:23 2015 -0700 ---------------------------------------------------------------------- .../spark/sql/catalyst/analysis/CheckAnalysis.scala | 10 ++++++++++ .../spark/sql/catalyst/analysis/AnalysisErrorSuite.scala | 5 +++++ 2 files changed, 15 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/6aceaf3d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index c23ab3c..81d473c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -103,6 +103,16 @@ trait CheckAnalysis { aggregateExprs.foreach(checkValidAggregateExpression) + case Sort(orders, _, _) => + orders.foreach { order => + order.dataType match { + case t: AtomicType => // OK + case NullType => // OK + case t => + failAnalysis(s"Sorting is not supported for columns of type ${t.simpleString}") + } + } + case _ => // Fallbacks to the following checks } http://git-wip-us.apache.org/repos/asf/spark/blob/6aceaf3d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala index 7bf678e..2588df9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala @@ -114,6 +114,11 @@ class AnalysisErrorSuite extends SparkFunSuite with BeforeAndAfter { "cannot cast" :: Literal(1).dataType.simpleString :: BinaryType.simpleString :: Nil) errorTest( + "sorting by unsupported column types", + listRelation.orderBy('list.asc), + "sorting" :: "type" :: "array<int>" :: Nil) + + errorTest( "non-boolean filters", testRelation.where(Literal(1)), "filter" :: "'1'" :: "not a boolean" :: Literal(1).dataType.simpleString :: Nil) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
