This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new e13278dc443c [SPARK-58273][SQL] Do not auto-fill generated columns for
by-position schema evolution inserts
e13278dc443c is described below
commit e13278dc443c6c447f602501c19941d9810099e9
Author: Szehon Ho <[email protected]>
AuthorDate: Thu Jul 23 10:44:22 2026 +0000
[SPARK-58273][SQL] Do not auto-fill generated columns for by-position
schema evolution inserts
### What changes were proposed in this pull request?
This PR updates `TableOutputResolver` so `INSERT WITH SCHEMA EVOLUTION` by
position does not auto-fill trailing generated columns. If a by-position insert
omits a trailing generated column, Spark now throws the same arity mismatch
error as a regular by-position insert.
The PR also adds coverage that:
- by-position `INSERT WITH SCHEMA EVOLUTION` rejects an omitted trailing
generated column;
- by-name `INSERT WITH SCHEMA EVOLUTION` with an explicit column list still
auto-fills the generated column.
### Why are the changes needed?
By-position writes require values for all target columns. Generated columns
are special because users can omit them only through by-name writes with an
explicit column list, where Spark can unambiguously identify the stored input
columns used by the generation expression.
Without this change, the by-position schema-evolution path could auto-fill
a trailing generated column, making it inconsistent with regular by-position
inserts and with Delta behavior.
### Does this PR introduce _any_ user-facing change?
Yes. In the current unreleased behavior, a by-position `INSERT WITH SCHEMA
EVOLUTION` could omit a trailing generated column and have Spark auto-fill it.
After this change, that insert fails with
`INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS`. Users can still omit
generated columns with a by-name insert and an explicit column list.
### How was this patch tested?
Added a regression test in `GeneratedColumnWriteSuite`.
Ran:
```
MAVEN_MIRROR_URL=https://maven-proxy.cloud.databricks.com build/sbt
'sql/testOnly org.apache.spark.sql.connector.GeneratedColumnWriteSuite -- -z
"INSERT WITH SCHEMA EVOLUTION by position does not auto-fill trailing generated
column"'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #57441 from szehon-ho/codex/generated-column-schema-evolution.
Authored-by: Szehon Ho <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
(cherry picked from commit 31f86edef847f24d6241fa170c1e8a3eb742dca6)
Signed-off-by: Uros Bojanic <[email protected]>
---
.../sql/catalyst/analysis/TableOutputResolver.scala | 12 +++++++++---
.../sql/connector/GeneratedColumnWriteSuite.scala | 20 ++++++++++++++++++++
2 files changed, 29 insertions(+), 3 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala
index e5bd9a2956c5..b50c77eee479 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TableOutputResolver.scala
@@ -43,9 +43,9 @@ object TableOutputResolver extends SQLConfHelper with Logging
{
/**
* Modes for filling in default or null values for missing columns.
* If FILL, fill missing top-level columns with their default values
(by-name reorder path).
- * If RECURSE, fill missing top-level columns (including trailing columns on
the by-position
- * path for INSERT with schema evolution when enabled) and recurse into
nested structs,
- * arrays, and maps to fill missing struct fields with null or defaults.
+ * If RECURSE, fill missing top-level columns (including trailing
non-generated columns on the
+ * by-position path for INSERT with schema evolution when enabled) and
recurse into nested
+ * structs, arrays, and maps to fill missing struct fields with null or
defaults.
* If NONE, do not fill any missing columns.
*/
object DefaultValueFillMode extends Enumeration {
@@ -541,6 +541,12 @@ object TableOutputResolver extends SQLConfHelper with
Logging {
}
val trailingCols = actualExpectedCols.drop(inputCols.size)
+ if (colPath.isEmpty &&
+ trailingCols.exists(col =>
GeneratedColumn.isGeneratedColumn(col.metadata))) {
+ throw QueryCompilationErrors.cannotWriteNotEnoughColumnsToTableError(
+ tableName, actualExpectedCols.map(_.name),
inputCols.map(_.toAttribute))
+ }
+
val defaults = if (fillDefaultValue || trailingCols.nonEmpty) {
trailingCols.map { expectedCol =>
// Check for generated column expression first, before falling back to
defaults.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
index 67d253ea01c5..54875c54a103 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/GeneratedColumnWriteSuite.scala
@@ -372,6 +372,26 @@ class GeneratedColumnWriteSuite extends QueryTest with
DatasourceV2SQLBase {
}
}
+ test("INSERT WITH SCHEMA EVOLUTION by position does not auto-fill trailing
generated column") {
+ val tblName = "my_tab"
+ withTable(s"testcat.$tblName") {
+ sql(s"""CREATE TABLE testcat.$tblName(
+ | id INT,
+ | doubled INT GENERATED ALWAYS AS (id * 2)
+ |) USING foo""".stripMargin)
+
+ withSQLConf(SQLConf.INSERT_INTO_NESTED_TYPE_COERCION_ENABLED.key ->
"true") {
+ val ex = intercept[AnalysisException] {
+ sql(s"INSERT WITH SCHEMA EVOLUTION INTO TABLE testcat.$tblName
SELECT 5")
+ }
+ assert(ex.getCondition ==
"INSERT_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS")
+ }
+
+ sql(s"INSERT WITH SCHEMA EVOLUTION INTO TABLE testcat.$tblName(id)
SELECT 5")
+ checkAnswer(spark.table(s"testcat.$tblName"), Row(5, 10))
+ }
+ }
+
test("INSERT by position with matching explicit value succeeds") {
val tblName = "my_tab"
withTable(s"testcat.$tblName") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
