This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 069f1c954012 [SPARK-54573][SQL][TESTS] Add test for CONSTRAINTS in
case of source with less nested struct field than target
069f1c954012 is described below
commit 069f1c95401289e9f2a67b5e5708249b868398df
Author: Szehon Ho <[email protected]>
AuthorDate: Thu Dec 4 08:53:28 2025 -0800
[SPARK-54573][SQL][TESTS] Add test for CONSTRAINTS in case of source with
less nested struct field than target
### What changes were proposed in this pull request?
Adding a test if CONSTRAINTS work in this case.
### Why are the changes needed?
Test-only, as this feature is going out in Spark 4.1.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Run the test
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53291 from szehon-ho/merge_nested_source_constraint_test.
Authored-by: Szehon Ho <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 081e5c74d5cbf36908ae7902393dc0ae46abffa6)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../sql/connector/MergeIntoTableSuiteBase.scala | 48 ++++++++++++++++++++++
1 file changed, 48 insertions(+)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/MergeIntoTableSuiteBase.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/MergeIntoTableSuiteBase.scala
index a36aeab1295a..e59b4435c408 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/MergeIntoTableSuiteBase.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/MergeIntoTableSuiteBase.scala
@@ -5354,6 +5354,54 @@ abstract class MergeIntoTableSuiteBase extends
RowLevelOperationSuiteBase
sql(s"DROP TABLE IF EXISTS $tableNameAsString")
}
+ test("merge with struct missing nested field with check constraint") {
+ withTempView("source") {
+ // Target table has struct with nested field c2
+ createAndInitTable(
+ s"""pk INT NOT NULL,
+ |s STRUCT<c1: INT, c2: INT>,
+ |dep STRING""".stripMargin,
+ """{ "pk": 0, "s": { "c1": 1, "c2": 10 }, "dep": "sales" }
+ |{ "pk": 1, "s": { "c1": 2, "c2": 20 }, "dep": "hr" }"""
+ .stripMargin)
+
+ // Add CHECK constraint on nested field c2 using ALTER TABLE
+ sql(s"ALTER TABLE $tableNameAsString ADD CONSTRAINT check_c2 CHECK " +
+ s"(s.c2 IS NOT NULL AND s.c2 > 1)")
+
+ // Source table schema with struct missing the c2 field
+ val sourceTableSchema = StructType(Seq(
+ StructField("pk", IntegerType),
+ StructField("s", StructType(Seq(
+ StructField("c1", IntegerType)
+ // missing field 'c2' which has CHECK constraint IS NOT NULL AND > 1
+ ))),
+ StructField("dep", StringType)
+ ))
+
+ val data = Seq(
+ Row(1, Row(100), "engineering"),
+ Row(2, Row(200), "finance")
+ )
+ spark.createDataFrame(spark.sparkContext.parallelize(data),
sourceTableSchema)
+ .createOrReplaceTempView("source")
+
+ withSQLConf(SQLConf.MERGE_INTO_NESTED_TYPE_COERCION_ENABLED.key ->
"true") {
+ val error = intercept[SparkRuntimeException] {
+ sql(
+ s"""MERGE INTO $tableNameAsString t USING source
+ |ON t.pk = source.pk
+ |WHEN MATCHED THEN
+ | UPDATE SET s = source.s, dep = source.dep
+ |""".stripMargin)}
+ assert(error.getCondition == "CHECK_CONSTRAINT_VIOLATION")
+ assert(error.getMessage.contains("CHECK constraint check_c2 s.c2 IS
NOT NULL AND " +
+ "s.c2 > 1 violated by row with values:\n - s.c2 : null"))
+ }
+ }
+ sql(s"DROP TABLE IF EXISTS $tableNameAsString")
+ }
+
test("merge with schema evolution using dataframe API: add new column and
set all") {
Seq(true, false).foreach { withSchemaEvolution =>
val sourceTable = "cat.ns1.source_table"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
