bhollis-dbx commented on code in PR #59283:
URL: https://github.com/apache/spark/pull/59283#discussion_r4300341711
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala:
##########
@@ -759,9 +737,214 @@ case class WithField(name: String, valExpr: Expression)
/**
* Drop a field by name.
*/
-case class DropField(name: String) extends StructFieldsOperation with
LeafLike[Expression] {
- override def apply(values: Seq[(StructField, Expression)]):
Seq[(StructField, Expression)] =
- values.filterNot { case (field, _) => resolver(field.name, name) }
+case class DropField(name: String) extends StructFieldsOperation with
LeafLike[Expression]
+
+/**
+ * Updates the current value of a field without duplicating the parent struct
expression.
+ */
+case class UpdateNestedField(name: String, fieldOps:
Seq[StructFieldsOperation])
+ extends StructFieldsOperation {
+
+ override def children: Seq[Expression] = fieldOps
+
+ override def prettyName: String = "WithField"
+
+ override protected def withNewChildrenInternal(
+ newChildren: IndexedSeq[Expression]): Expression =
+ copy(fieldOps = newChildren.map(_.asInstanceOf[StructFieldsOperation]))
+}
+
+/**
+ * Builds a struct from fields copied from `structExpr` and values computed by
`updatedValues`.
+ *
+ * `outputOrdinals` describes each output position. A nonnegative entry copies
that field from
+ * `structExpr`; a negative entry evaluates the next expression in
`updatedValues`. Storing copied
+ * fields as ordinals avoids adding a `GetStructField` child for every copied
field. Building rows
+ * still visits every output field.
+ *
+ * For example, to transform `(a, b, c)` into `(a, newB, c, newD)`,
`outputOrdinals` is
+ * `Seq(0, -1, 2, -1)` and `updatedValues` is `Seq(newB, newD)`. The first and
third output fields
+ * copy source fields 0 and 2. The negative entries use `newB` and `newD`, in
that order.
+ *
+ * If a nullable `structExpr` evaluates to null, the result is null and
`updatedValues` are not
+ * evaluated.
+ */
+private case class UpdateFieldsExpression(
+ structExpr: Expression,
+ updatedValues: Seq[Expression],
+ outputOrdinals: Seq[Int],
+ outputFields: Seq[StructField]) extends ConditionalExpression {
+
+ override def children: Seq[Expression] = structExpr +: updatedValues
Review Comment:
Fixed in 85541e385. Non-nullable updates that replace every field now lower
to constructors, removing unused inputs and nested With definitions. Added
regressions for discarded aggregates and preserved null behavior.
Both pruning regressions fail before the fix. All 19 Catalyst tests pass
using cached dependencies. The SQL execution regression remains blocked locally
by incompatible cached classes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
