dtenedor commented on code in PR #48940:
URL: https://github.com/apache/spark/pull/48940#discussion_r1859076220
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala:
##########
@@ -5978,7 +5982,56 @@ class AstBuilder extends DataTypeAstBuilder
withQueryResultClauses(c, withSubqueryAlias(), forPipeOperators = true)
}.getOrElse(
visitOperatorPipeAggregate(ctx, left)
- )))))))))
+ ))))))))))
+ }
+
+ private def visitOperatorPipeSet(
+ ctx: OperatorPipeRightSideContext, left: LogicalPlan): LogicalPlan = {
+ val (setIdentifiers: Seq[String], setTargets: Seq[Expression]) =
+ visitOperatorPipeSetAssignmentSeq(ctx.operatorPipeSetAssignmentSeq())
+ var plan = left
+ val visitedSetIdentifiers = mutable.Set.empty[String]
+ setIdentifiers.zip(setTargets).foreach {
+ case (_, _: Alias) =>
+ operationNotAllowed(
+ "SQL pipe syntax |> SET operator with an alias assigned with [AS]
aliasName", ctx)
+ case (ident, target) =>
+ // Check uniqueness of the assignment keys.
+ val checkKey = if (SQLConf.get.caseSensitiveAnalysis) {
+ ident.toLowerCase(Locale.ROOT)
+ } else {
+ ident
+ }
+ if (visitedSetIdentifiers(checkKey)) {
+ operationNotAllowed(
+ s"SQL pipe syntax |> SET operator with duplicate assignment key
$ident", ctx)
+ }
+ visitedSetIdentifiers += checkKey
+ // Add an UnresolvedStarExceptOrReplace to exclude the SET expression
name from the relation
+ // and add the new SET expression to the projection list.
+ // Use a PipeSelect expression to make sure it does not contain any
aggregate functions.
+ val replacement =
+ Alias(PipeExpression(target, isAggregate = false,
PipeOperators.setClause), ident)()
+ val projectList: Seq[NamedExpression] =
+ Seq(UnresolvedStarExceptOrReplace(
+ target = None, excepts = Seq(Seq(ident)), replacements =
Some(Seq(replacement))))
+ plan = Project(projectList, plan)
Review Comment:
Good question, yes, I found this was necessary when experimenting with
different test cases using the `SELECT * EXCEPT` implementation. I added a
comment here to explain this, and a couple new test cases as well to exercise
it (one with your example, another with an LCA). The optimizer will flatten the
projections in the following step.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
