peter-toth commented on code in PR #45343:
URL: https://github.com/apache/spark/pull/45343#discussion_r1521535582
##########
sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala:
##########
@@ -498,4 +559,70 @@ class DataFrameSelfJoinSuite extends QueryTest with
SharedSparkSession {
assert(df1.join(df2, $"t1.i" === $"t2.i").cache().count() == 1)
}
}
+
+ test("SPARK_47217: deduplication of project causes ambiguity in resolution")
{
+ val df = Seq((1, 2)).toDF("a", "b")
+ val df2 = df.select(df("a").as("aa"), df("b").as("bb"))
+ val df3 = df2.join(df, df2("bb") === df("b")).select(df2("aa"), df("a"))
+ checkAnswer(
+ df3,
+ Row(1, 1) :: Nil)
+ }
+
+ test("SPARK-47217. deduplication in nested joins with join attribute
aliased") {
+ val df1 = Seq((1, 2)).toDF("a", "b")
+ val df2 = Seq((1, 2)).toDF("aa", "bb")
+ val df1Joindf2 = df1.join(df2, df1("a") ===
df2("aa")).select(df1("a").as("aaa"),
+ df2("aa"), df1("b"))
+
+ assertCorrectResolution(df1Joindf2.join(df1, df1Joindf2("aaa") ===
df1("a")),
+ Resolution.LeftConditionToLeftLeg, Resolution.RightConditionToRightLeg)
+
+ assertCorrectResolution(df1.join(df1Joindf2, df1Joindf2("aaa") ===
df1("a")),
+ Resolution.LeftConditionToRightLeg, Resolution.RightConditionToLeftLeg)
+
+ val proj1 = df1Joindf2.join(df1, df1Joindf2("aaa") ===
df1("a")).select(df1Joindf2("aa"),
+ df1("a")).queryExecution.analyzed.asInstanceOf[Project]
+ val join1 = proj1.child.asInstanceOf[Join]
+ assert(proj1.projectList(0).references.subsetOf(join1.left.outputSet))
+ assert(proj1.projectList(1).references.subsetOf(join1.right.outputSet))
+
+ val proj2 = df1.join(df1Joindf2, df1Joindf2("aaa") ===
df1("a")).select(df1Joindf2("aa"),
+ df1("a")).queryExecution.analyzed.asInstanceOf[Project]
+ val join2 = proj2.child.asInstanceOf[Join]
+ assert(proj2.projectList(0).references.subsetOf(join2.right.outputSet))
+ assert(proj2.projectList(1).references.subsetOf(join2.left.outputSet))
+ }
+
+ test("SPARK-47217. deduplication in nested joins without join attribute
aliased") {
+ val df1 = Seq((1, 2)).toDF("a", "b")
+ val df2 = Seq((1, 2)).toDF("aa", "bb")
+ val df1Joindf2 = df1.join(df2, df1("a") === df2("aa")).select(df1("a"),
df2("aa"), df1("b"))
+
+ assertCorrectResolution(df1Joindf2.join(df1, df1Joindf2("a") === df1("a")),
+ Resolution.LeftConditionToLeftLeg, Resolution.RightConditionToRightLeg)
+
+ assertCorrectResolution(df1.join(df1Joindf2, df1Joindf2("a") === df1("a")),
+ Resolution.LeftConditionToRightLeg, Resolution.RightConditionToLeftLeg)
+
+ val proj1 = df1Joindf2.join(df1, df1Joindf2("a") ===
df1("a")).select(df1Joindf2("a"),
+ df1("a")).queryExecution.analyzed.asInstanceOf[Project]
Review Comment:
> I am looking at it from Point of View of user, that is one is coming from
df1("a") and another from df1Joindf2("a"), so there is no ambiguity either in
Join or Select
I agree that there is no ambiguity in `Join` becuase there we can suppose
that the user would like to build a relation between the left and right sides
(so one side of the `===` must come from left and the other from the right
side). This is why the following join with `df1("a") === df1("a")` also works:
```
val df1 = Seq((1, 2)).toDF("a", "b")
val df2 = Seq((1, 2)).toDF("aa", "bb")
val df1Joindf2 = df1.join(df2, df1("a") === df2("aa")).select(df1("a"),
df2("aa"), df1("b"))
val df4 = df1Joindf2.join(df1, df1("a") === df1("a"))
df4.explain(true)
```
But in a select on the join result using `df1("a")` should be ambigous as
`df1("a")` could be selected from both legs of the join. I.e. both
`df1Joindf2.select(df1("a"))` and `df1.select(df1("a"))` work.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
