[
https://issues.apache.org/jira/browse/SPARK-6430?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Michael Armbrust updated SPARK-6430:
------------------------------------
Target Version/s: 1.3.1
> Cannot resolve column correctlly when using left semi join
> ----------------------------------------------------------
>
> Key: SPARK-6430
> URL: https://issues.apache.org/jira/browse/SPARK-6430
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 1.3.0
> Environment: Spark 1.3.0 on yarn mode
> Reporter: zzc
>
> My code:
> {quote}
> case class TestData(key: Int, value: String)
> case class TestData2(a: Int, b: Int)
> import org.apache.spark.sql.execution.joins._
> import sqlContext.implicits._
> val testData = sc.parallelize(
> (1 to 100).map(i => TestData(i, i.toString))).toDF()
> testData.registerTempTable("testData")
> val testData2 = sc.parallelize(
> TestData2(1, 1) ::
> TestData2(1, 2) ::
> TestData2(2, 1) ::
> TestData2(2, 2) ::
> TestData2(3, 1) ::
> TestData2(3, 2) :: Nil, 2).toDF()
> testData2.registerTempTable("testData2")
> //val tmp = sqlContext.sql("SELECT * FROM testData *LEFT SEMI JOIN* testData2
> ON key = a ")
> val tmp = sqlContext.sql("SELECT testData2.b, count(testData2.b) FROM
> testData *LEFT SEMI JOIN* testData2 ON key = testData2.a group by
> testData2.b")
> tmp.explain()
> {quote}
> Error log:
> {quote}
> org.apache.spark.sql.AnalysisException: cannot resolve 'testData2.b' given
> input columns key, value; line 1 pos 108
> at
> org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$apply$3$$anonfun$apply$1.applyOrElse(CheckAnalysis.scala:48)
> at
> org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$apply$3$$anonfun$apply$1.applyOrElse(CheckAnalysis.scala:45)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:250)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:250)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:50)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:249)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$transformExpressionUp$1(QueryPlan.scala:103)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2$$anonfun$apply$2.apply(QueryPlan.scala:117)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> at
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> at scala.collection.immutable.List.foreach(List.scala:318)
> {quote}
> {quote}SELECT * FROM testData LEFT SEMI JOIN testData2 ON key = a{quote} is
> correct,
> {quote}
> SELECT a FROM testData LEFT SEMI JOIN testData2 ON key = a
> SELECT max(value) FROM testData LEFT SEMI JOIN testData2 ON key = a group by b
> SELECT max(value) FROM testData LEFT SEMI JOIN testData2 ON key = testData2.a
> group by testData2.b
> SELECT testData2.b, count(testData2.b) FROM testData LEFT SEMI JOIN testData2
> ON key = testData2.a group by testData2.b
> {quote} are incorrect.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
