This is an automated email from the ASF dual-hosted git repository.
zsxwing pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 07e110782166 [SPARK-58772][SQL] Make SelectedField pass column metadata
07e110782166 is described below
commit 07e11078216650767b2db781f52dbccf7b0b973e
Author: Shixiong Zhu <[email protected]>
AuthorDate: Sun Aug 16 21:22:15 2026 -0700
[SPARK-58772][SQL] Make SelectedField pass column metadata
### What changes were proposed in this pull request?
Update `SelectedField` to pass metadata to `StructField` objects when
pruning columns.
### Why are the changes needed?
When SelectedField prunes nested columns, it doesn't copy column metadata
at the following two places:
https://github.com/apache/spark/blob/042ad7d0c4ac1c4d3e9fdeb48e2695fdeb861135/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala#L88
https://github.com/apache/spark/blob/042ad7d0c4ac1c4d3e9fdeb48e2695fdeb861135/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala#L112
This can cause column metadata to be dropped and downstream cannot see the
column metadata.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Added two new unit test to verify `SelectedField` does keep metadata.
### Was this patch authored or co-authored using generative AI tooling?
Yep. Generated by Claude Code (Opus 4.8)
Closes #58007 from zsxwing/selectedfield-metadata.
Authored-by: Shixiong Zhu <[email protected]>
Signed-off-by: Shixiong Zhu <[email protected]>
(cherry picked from commit 5460c109699fa7f62c0e8796c996dda0f7ec8ccc)
Signed-off-by: Shixiong Zhu <[email protected]>
---
.../sql/catalyst/expressions/SelectedField.scala | 4 +--
.../catalyst/expressions/SelectedFieldSuite.scala | 37 ++++++++++++++++++++++
2 files changed, 39 insertions(+), 2 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala
index 69dfdbfc9a08..808083e93208 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SelectedField.scala
@@ -85,7 +85,7 @@ object SelectedField {
expr match {
case a: Attribute =>
dataTypeOpt.map { dt =>
- StructField(a.name, dt, a.nullable)
+ StructField(a.name, dt, a.nullable, a.metadata)
}
case c: GetStructField =>
val field = c.childSchema(c.ordinal)
@@ -109,7 +109,7 @@ object SelectedField {
// This should not happen.
throw QueryCompilationErrors.dataTypeUnsupportedByClassError(x,
"GetArrayStructFields")
}
- val newField = StructField(field.name, newFieldDataType,
field.nullable)
+ val newField = StructField(field.name, newFieldDataType,
field.nullable, field.metadata)
selectField(child, Option(ArrayType(struct(newField), containsNull)))
case GetMapValue(child, key) if key.foldable =>
// GetMapValue does not select a field from a struct (i.e. prune the
struct) so it can't be
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
index ddeac6cbb933..8db237e1a68c 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
@@ -524,6 +524,43 @@ class SelectedFieldSuite extends AnalysisTest {
StructField("subfield4", IntegerType) :: Nil)) :: Nil), containsNull
= false)))
}
+ // |-- col1: string (nullable = false)
+ // |-- col2: struct (nullable = true) (metadata = {"outer":"meta"})
+ // | |-- field1: long (nullable = true) (metadata = {"inner":"meta"})
+ // | |-- field2: long (nullable = true)
+ private val outerMetadata = new MetadataBuilder().putString("outer",
"meta").build()
+ private val innerMetadata = new MetadataBuilder().putString("inner",
"meta").build()
+ private val structWithMetadata = StructType(ignoredField ::
+ StructField("col2", StructType(
+ StructField("field1", LongType, nullable = true, innerMetadata) ::
+ StructField("field2", LongType) :: Nil), nullable = true, outerMetadata)
:: Nil)
+
+ testSelect(structWithMetadata, "col2.field1") {
+ StructField("col2", StructType(
+ StructField("field1", LongType, nullable = true, innerMetadata) :: Nil),
+ nullable = true, outerMetadata)
+ }
+
+ // |-- col1: string (nullable = false)
+ // |-- col2: struct (nullable = true)
+ // | |-- field3: array (nullable = false)
+ // | | |-- element: struct (containsNull = true)
+ // | | | |-- subfield1: long (nullable = true) (metadata =
{"elem":"meta"})
+ // | | | |-- subfield2: long (nullable = true)
+ private val elementMetadata = new MetadataBuilder().putString("elem",
"meta").build()
+ private val arrayOfStructWithMetadata = StructType(ignoredField ::
+ StructField("col2", StructType(
+ StructField("field3", ArrayType(StructType(
+ StructField("subfield1", LongType, nullable = true, elementMetadata) ::
+ StructField("subfield2", LongType) :: Nil)), nullable = false) ::
Nil)) :: Nil)
+
+ testSelect(arrayOfStructWithMetadata, "col2.field3.subfield1") {
+ StructField("col2", StructType(
+ StructField("field3", ArrayType(StructType(
+ StructField("subfield1", LongType, nullable = true, elementMetadata)
:: Nil)),
+ nullable = false) :: Nil))
+ }
+
def assertResult(expected: StructField)(actual: StructField)(selectExpr:
String): Unit = {
try {
super.assertResult(expected)(actual)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
