This is an automated email from the ASF dual-hosted git repository.

pan3793 pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 30e71be8e2e1 [SPARK-58400][SQL] Fix ANSI mode assumption for views 
created by Spark 4.0+
30e71be8e2e1 is described below

commit 30e71be8e2e1d757bf9fbb64547433ea4a60fc34
Author: Cheng Pan <[email protected]>
AuthorDate: Wed Jul 29 09:01:00 2026 -0700

    [SPARK-58400][SQL] Fix ANSI mode assumption for views created by Spark 4.0+
    
    ### What changes were proposed in this pull request?
    
    1. `Analyzer.trySetAnsiValue`: replace 
`createSparkVersion.startsWith("4.")` with 
`VersionUtils.majorMinorPatchVersion(createSparkVersion).exists { case (major, 
_, _) => major >= 4 }`. The old check only matched the 4.x major; views created 
by Spark 5.x (ANSI still the default) wrongly fell to ANSI=false.
    2. View resolution now forwards `createSparkVersion` to 
`View.effectiveSQLConf` on all paths that build the throwaway conf from 
persisted `viewSQLConfigs` -- `ViewResolver.resolve` (single-pass) and 
`SessionCatalog.parseMetricViewDefinition` -- matching 
`SessionCatalog.fromCatalogTable` / the fixed-point `ViewResolution.resolve`, 
which already forwarded it.
    
    ### Why are the changes needed?
    
    ANSI became the default in 4.0 (SPARK-44444) and remains so in 5.x. 
`trySetAnsiValue`'s `startsWith("4.")` only recognized 4.x, so a 5.x-created 
view with no persisted ANSI value was resolved as LEGACY instead of ANSI -- its 
schema-enforcement casts used the wrong eval mode.
    
    The single-pass `ViewResolver` and `parseMetricViewDefinition` compounded 
this by dropping `createVersion` entirely. This surfaced as a dual-run 
`LOGICAL_PLAN_COMPARISON_MISMATCH`: the two analyzers' plans differed only in 
`Cast.evalMode`, which `NormalizePlan` does not strip. On master the failing 
test passes only by coincidence (`"5.0.0"` and `""` both miss 
`startsWith("4.")` and both resolve to LEGACY).
    
    ### Does this PR introduce _any_ user-facing change?
    
    Bug fix. Views created by Spark 4.0+ without a persisted ANSI value are now 
resolved with ANSI=true (matching their creating version's default) in both 
analyzers. Empty/unparseable `createVersion` still maps to ANSI=false, as 
documented.
    
    ### How was this patch tested?
    
    Added `AlwaysPersistedConfigsSuite` -> "ANSI value derived from 
createSparkVersion when not persisted for views", exercising 
`View.effectiveSQLConf`/`trySetAnsiValue` for `createSparkVersion` in 
`{"3.5.0", "4.0.0", "4.0.0-SNAPSHOT", "5.0.0", "", "bogus"}` (ANSI true for 
major `>= 4`, false otherwise/unparseable). Existing suites: 
`HiveSQLViewSuite`, `SQLViewSuite`, `ViewResolverSuite`, `HybridAnalyzerSuite`, 
`MetricViewSuite`, `MetricViewV2CatalogSuite`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Yes. Generated-by: GLM 5.2
    
    Closes #57593 from pan3793/ansi-view-fix.
    
    Authored-by: Cheng Pan <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../apache/spark/sql/catalyst/analysis/Analyzer.scala | 16 ++++++++--------
 .../sql/catalyst/analysis/resolver/ViewResolver.scala |  5 ++++-
 .../spark/sql/catalyst/catalog/SessionCatalog.scala   |  3 ++-
 .../spark/sql/AlwaysPersistedConfigsSuite.scala       | 19 +++++++++++++++++++
 4 files changed, 33 insertions(+), 10 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 01513d8926c1..ec17902e2e85 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -67,6 +67,7 @@ import org.apache.spark.sql.internal.connector.V1Function
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.ArrayImplicits._
+import org.apache.spark.util.VersionUtils
 
 /**
  * A trivial [[Analyzer]] with a dummy [[SessionCatalog]] and
@@ -314,18 +315,17 @@ object Analyzer {
 
   /**
    * In case ANSI value wasn't persisted for a view or a UDF, we set it to 
`true` in case Spark
-   * version used to create the view is 4.0.0 or higher. We set it to `false` 
in case Spark version
-   * is lower than 4.0.0 or if the Spark version wasn't stored (in that case 
we assume that the
-   * value is `false`)
+   * version used to create the view is 4.0.0 or higher (ANSI SQL mode became 
the default in
+   * Spark 4.0, see SPARK-44444). We set it to `false` in case Spark version 
is lower than 4.0.0
+   * or if the Spark version wasn't stored / can't be parsed (in that case we 
assume that the
+   * value is `false`).
    */
   def trySetAnsiValue(sqlConf: SQLConf, createSparkVersion: String = ""): Unit 
= {
     if (conf.getConf(SQLConf.ASSUME_ANSI_FALSE_IF_NOT_PERSISTED) &&
       !sqlConf.settings.containsKey(SQLConf.ANSI_ENABLED.key)) {
-      if (createSparkVersion.startsWith("4.")) {
-        sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, "true")
-      } else {
-        sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, "false")
-      }
+      val assumeAnsiEnabled = 
VersionUtils.majorMinorPatchVersion(createSparkVersion)
+        .exists { case (major, _, _) => major >= 4 }
+      sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, 
assumeAnsiEnabled.toString)
     }
   }
 
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
index a224e521b548..8aa5262c4b59 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
@@ -102,7 +102,10 @@ class ViewResolver(
 
     val (resolvedChild, usedViewResolutionContext) = 
withViewResolutionContext(unresolvedView) {
       SQLConf.withExistingConf(
-        View.effectiveSQLConf(unresolvedView.desc.viewSQLConfigs, 
unresolvedView.isTempView)
+        View.effectiveSQLConf(
+          configs = unresolvedView.desc.viewSQLConfigs,
+          isTempView = unresolvedView.isTempView,
+          createSparkVersion = unresolvedView.desc.createVersion)
       ) {
         checkResolverGuard(unresolvedView)
 
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 37e03e2284d2..9a08cf1ee49e 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -1153,7 +1153,8 @@ class SessionCatalog(
     SQLConf.withExistingConf(
       View.effectiveSQLConf(
         configs = viewConfigs,
-        isTempView = false
+        isTempView = false,
+        createSparkVersion = metadata.createVersion
       )
     ) {
       CurrentOrigin.withOrigin(origin) {
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/AlwaysPersistedConfigsSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/AlwaysPersistedConfigsSuite.scala
index 5612c831c37a..634eb2b0c028 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/AlwaysPersistedConfigsSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/AlwaysPersistedConfigsSuite.scala
@@ -173,6 +173,25 @@ class AlwaysPersistedConfigsSuite extends 
SharedSparkSession {
     assert(sqlConf.settings.get("spark.sql.ansi.enabled") == "false")
   }
 
+  test("SPARK-58400: ANSI value derived from createSparkVersion when not 
persisted for views") {
+    Seq(
+      "3.5.0" -> false,
+      "4.0.0" -> true,
+      "4.0.0-SNAPSHOT" -> true,
+      "5.0.0" -> true,
+      "" -> false,
+      "bogus" -> false
+    ).foreach { case (version, expected) =>
+      val sqlConf = View.effectiveSQLConf(
+        configs = Map.empty[String, String],
+        isTempView = false,
+        createSparkVersion = version)
+
+      assert(sqlConf.settings.get("spark.sql.ansi.enabled") == 
expected.toString,
+        s"createSparkVersion='$version'")
+    }
+  }
+
   test("Current schema marker is materialized in persisted view path") {
     withView(testViewName) {
       withSQLConf(SQLConf.PATH_ENABLED.key -> "true") {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to