cloud-fan commented on code in PR #44689:
URL: https://github.com/apache/spark/pull/44689#discussion_r1449707533
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala:
##########
@@ -696,6 +696,37 @@ case class ResolvedStar(expressions: Seq[NamedExpression])
extends Star with Une
override def toString: String = expressions.mkString("ResolvedStar(", ", ",
")")
}
+/**
+ * Represents all input attributes to a given relational operator.
+ * This is used in Spark Connect dataframe, for example:
+ * df1 = spark.createDataFrame([{"id": 1}])
+ * df2 = spark.createDataFrame([{"id": 1, "val": "v"}])
+ * df1.join(df2, "id").select(df1["*"])
+ * @param planId the plan id of target node.
+ */
+case class UnresolvedDataFrameStar(planId: Long) extends Star with Unevaluable
{
+ override def expand(input: LogicalPlan, resolver: Resolver):
Seq[NamedExpression] = {
+ val resolved = resolveDFStarRecursively(planId, input)
+ resolved.map(_.expand(input, resolver)).getOrElse(
+ throw QueryCompilationErrors.cannotResolveStar(this)
+ )
+ }
+
+ private def resolveDFStarRecursively(
+ id: Long,
+ p: LogicalPlan): Option[ResolvedStar] = {
+ val resolved = if (p.getTagValue(LogicalPlan.PLAN_ID_TAG).contains(id)) {
+ Some(ResolvedStar(p.output))
+ } else {
+ p.children.iterator.map(resolveDFStarRecursively(id, _))
+ .foldLeft(Option.empty[ResolvedStar]) {
Review Comment:
It's probably a bug in vanilla spark...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
