Repository: spark
Updated Branches:
refs/heads/branch-1.3 bc92a2e40 -> f0141ca38
[SPARK-6459][SQL] Warn when constructing trivially true equals predicate
For example, one might expect the following code to work, but it does not. Now
you will at least get a warning with a suggestion to use aliases.
```scala
val df = sqlContext.load(path, "parquet")
val txns = df.groupBy("cust_id").agg($"cust_id",
countDistinct($"day_num").as("txns"))
val spend = df.groupBy("cust_id").agg($"cust_id",
sum($"extended_price").as("spend"))
val rmJoin = txns.join(spend, txns("cust_id") === spend("cust_id"), "inner")
```
Author: Michael Armbrust <[email protected]>
Closes #5163 from marmbrus/selfJoinError and squashes the following commits:
16c1f0b [Michael Armbrust] fix visibility
1b57e8d [Michael Armbrust] Warn when constructing trivially true equals
predicate
(cherry picked from commit 32efadd0500f10bddf2ae8456c9e719ec52940f1)
Signed-off-by: Michael Armbrust <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f0141ca3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f0141ca3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f0141ca3
Branch: refs/heads/branch-1.3
Commit: f0141ca385047550594541703809f5ddf75b480a
Parents: bc92a2e
Author: Michael Armbrust <[email protected]>
Authored: Tue Mar 24 12:09:02 2015 -0700
Committer: Michael Armbrust <[email protected]>
Committed: Tue Mar 24 12:09:11 2015 -0700
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/sql/Column.scala | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f0141ca3/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index ec7d15f..1f5c30b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql
import scala.language.implicitConversions
import org.apache.spark.annotation.Experimental
+import org.apache.spark.Logging
import org.apache.spark.sql.functions.lit
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute,
UnresolvedStar, UnresolvedGetField}
@@ -46,7 +47,7 @@ private[sql] object Column {
* @groupname Ungrouped Support functions for DataFrames.
*/
@Experimental
-class Column(protected[sql] val expr: Expression) {
+class Column(protected[sql] val expr: Expression) extends Logging {
def this(name: String) = this(name match {
case "*" => UnresolvedStar(None)
@@ -109,7 +110,15 @@ class Column(protected[sql] val expr: Expression) {
*
* @group expr_ops
*/
- def === (other: Any): Column = EqualTo(expr, lit(other).expr)
+ def === (other: Any): Column = {
+ val right = lit(other).expr
+ if (this.expr == right) {
+ logWarning(
+ s"Constructing trivially true equals predicate, '${this.expr} =
$right'. " +
+ "Perhaps you need to use aliases.")
+ }
+ EqualTo(expr, right)
+ }
/**
* Equality test.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
