This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 7012a923fd81 [SPARK-54578][SQL] Perform Code Cleanup on
`AssignmentUtils`
7012a923fd81 is described below
commit 7012a923fd81c9cf5740670333241627a95de4fc
Author: yangjie01 <[email protected]>
AuthorDate: Thu Dec 4 08:50:25 2025 -0800
[SPARK-54578][SQL] Perform Code Cleanup on `AssignmentUtils`
### What changes were proposed in this pull request?
This PR performs a code cleanup on `AssignmentUtils`:
1. Deleted the redundant private methods `getMissingSourcePaths` and
`createNullCheckForFieldPath`, which were introduced in
https://github.com/apache/spark/pull/53149 but became obsolete after
https://github.com/apache/spark/pull/53229.
2. Removed the redundant documentation for the invalid input parameter
`missingSourcePaths` in the `alignUpdateAssignments` method.
3. Removed the unused input parameter `updateStar` from the private method
`applyAssignments`.
4. Corrected a spelling error in the input parameter `coerceNestedTypes` of
the private method `applyFieldAssignments`.
### Why are the changes needed?
Code Cleanup.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- Pass Github Actions
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53302 from LuciferYang/SPARK-54578.
Lead-authored-by: yangjie01 <[email protected]>
Co-authored-by: YangJie <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 8b5d900e0f7899dd17b91b6e5293eaca7947ee4e)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../sql/catalyst/analysis/AssignmentUtils.scala | 59 ++--------------------
1 file changed, 5 insertions(+), 54 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala
index c9ed5b86dbde..6c7b0626e81e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
import org.apache.spark.sql.catalyst.SQLConfHelper
import
org.apache.spark.sql.catalyst.analysis.TableOutputResolver.DefaultValueFillMode.{NONE,
RECURSE}
-import org.apache.spark.sql.catalyst.expressions.{Attribute,
CreateNamedStruct, Expression, GetStructField, IsNull, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Attribute,
CreateNamedStruct, Expression, GetStructField, Literal}
import org.apache.spark.sql.catalyst.plans.logical.Assignment
import org.apache.spark.sql.catalyst.types.DataTypeUtils
import org.apache.spark.sql.catalyst.util.CharVarcharUtils
@@ -55,7 +55,6 @@ object AssignmentUtils extends SQLConfHelper with CastSupport
{
* (preserving existing fields).
* @param coerceNestedTypes whether to coerce nested types to match the
target type
* for complex types
- * @param missingSourcePaths paths that exist in target but not in source
* @return aligned update assignments that match table attributes
*/
def alignUpdateAssignments(
@@ -73,8 +72,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport
{
assignments,
addError = err => errors += err,
colPath = Seq(attr.name),
- coerceNestedTypes,
- fromStar)
+ coerceNestedTypes)
}
if (errors.nonEmpty) {
@@ -158,8 +156,7 @@ object AssignmentUtils extends SQLConfHelper with
CastSupport {
assignments: Seq[Assignment],
addError: String => Unit,
colPath: Seq[String],
- coerceNestedTypes: Boolean = false,
- updateStar: Boolean = false): Expression = {
+ coerceNestedTypes: Boolean = false): Expression = {
val (exactAssignments, otherAssignments) = assignments.partition {
assignment =>
assignment.key.semanticEquals(colExpr)
@@ -197,7 +194,7 @@ object AssignmentUtils extends SQLConfHelper with
CastSupport {
assignments: Seq[Assignment],
addError: String => Unit,
colPath: Seq[String],
- coerceNestedTyptes: Boolean): Expression = {
+ coerceNestedTypes: Boolean): Expression = {
col.dataType match {
case structType: StructType =>
@@ -207,7 +204,7 @@ object AssignmentUtils extends SQLConfHelper with
CastSupport {
}
val updatedFieldExprs = fieldAttrs.zip(fieldExprs).map { case
(fieldAttr, fieldExpr) =>
applyAssignments(fieldAttr, fieldExpr, assignments, addError,
colPath :+ fieldAttr.name,
- coerceNestedTyptes)
+ coerceNestedTypes)
}
toNamedStruct(structType, updatedFieldExprs)
@@ -226,52 +223,6 @@ object AssignmentUtils extends SQLConfHelper with
CastSupport {
CreateNamedStruct(namedStructExprs)
}
- private def getMissingSourcePaths(targetType: StructType,
- sourceType: DataType,
- colPath: Seq[String],
- addError: String => Unit): Seq[Seq[String]] =
{
- val nestedTargetPaths = DataTypeUtils.extractLeafFieldPaths(targetType,
Seq.empty)
- val nestedSourcePaths = sourceType match {
- case sourceStructType: StructType =>
- DataTypeUtils.extractLeafFieldPaths(sourceStructType, Seq.empty)
- case _ =>
- addError(s"Value for struct type: " +
- s"${colPath.quoted} must be a struct but was
${sourceType.simpleString}")
- Seq()
- }
- nestedSourcePaths.diff(nestedTargetPaths)
- }
-
- /**
- * Creates a null check for a field at the given path within a struct
expression.
- * Navigates through the struct hierarchy following the path and returns an
IsNull check
- * for the final field.
- *
- * @param rootExpr the root expression to navigate from
- * @param path the field path to navigate (sequence of field names)
- * @return an IsNull expression checking if the field at the path is null
- */
- private def createNullCheckForFieldPath(
- rootExpr: Expression,
- path: Seq[String]): Expression = {
- var currentExpr: Expression = rootExpr
- path.foreach { fieldName =>
- currentExpr.dataType match {
- case st: StructType =>
- st.fields.find(f => conf.resolver(f.name, fieldName)) match {
- case Some(field) =>
- val fieldIndex = st.fieldIndex(field.name)
- currentExpr = GetStructField(currentExpr, fieldIndex,
Some(field.name))
- case None =>
- // Field not found, shouldn't happen
- }
- case _ =>
- // Not a struct, shouldn't happen
- }
- }
- IsNull(currentExpr)
- }
-
/**
* Checks whether assignments are aligned and compatible with table columns.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
