[spark] branch branch-3.1 updated: [SPARK-34613][SQL] Fix view does not capture disable hint config

[wenchen]

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
     new b73b1dc  [SPARK-34613][SQL] Fix view does not capture disable hint 
config
b73b1dc is described below

commit b73b1dca54ec95d2085eb6d5daf659d695fea1a5
Author: ulysses-you <ulyssesyo...@gmail.com>
AuthorDate: Fri Mar 5 12:19:30 2021 +0800

    [SPARK-34613][SQL] Fix view does not capture disable hint config
    
    ### What changes were proposed in this pull request?
    
    Add allow list to capture sql config for view.
    
    ### Why are the changes needed?
    
    Spark use origin text sql to store view then capture and store sql config 
into view metadata.
    
    Capture config will skip some config with some prefix, e.g. 
`spark.sql.optimizer.` but unfortunately `spark.sql.optimizer.disableHints` is 
start with `spark.sql.optimizer.`.
    
    We need a allow list to help capture the config.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes bug fix.
    
    ### How was this patch tested?
    
    Add test.
    
    Closes #31732 from ulysses-you/SPARK-34613.
    
    Authored-by: ulysses-you <ulyssesyo...@gmail.com>
    Signed-off-by: Wenchen Fan <wenc...@databricks.com>
    (cherry picked from commit 43aacd5069294e1215e86cd43bd0810bda998be2)
    Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
 .../org/apache/spark/sql/execution/command/views.scala    | 12 +++++++++++-
 .../org/apache/spark/sql/execution/SQLViewTestSuite.scala | 15 +++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 960fe4a..ef0e90d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -353,8 +353,18 @@ object ViewHelper {
     "spark.sql.shuffle.",
     "spark.sql.adaptive.")
 
+  private val configAllowList = Seq(
+    SQLConf.DISABLE_HINTS.key
+  )
+
+  /**
+   * Capture view config either of:
+   * 1. exists in allowList
+   * 2. do not exists in denyList
+   */
   private def shouldCaptureConfig(key: String): Boolean = {
-    !configPrefixDenyList.exists(prefix => key.startsWith(prefix))
+    configAllowList.exists(prefix => key.equals(prefix)) ||
+      !configPrefixDenyList.exists(prefix => key.startsWith(prefix))
   }
 
   import CatalogTable._
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
index 84a20bb..88218b1 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution
 
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.plans.logical.Repartition
 import org.apache.spark.sql.internal.SQLConf._
 import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
 
@@ -278,6 +279,20 @@ abstract class SQLViewTestSuite extends QueryTest with 
SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-34613: Fix view does not capture disable hint config") {
+    withSQLConf(DISABLE_HINTS.key -> "true") {
+      val viewName = createView("v1", "SELECT /*+ repartition(1) */ 1")
+      withView(viewName) {
+        assert(
+          sql(s"SELECT * FROM $viewName").queryExecution.analyzed.collect {
+            case e: Repartition => e
+          }.isEmpty
+        )
+        checkViewOutput(viewName, Seq(Row(1)))
+      }
+    }
+  }
 }
 
 class LocalTempViewTestSuite extends SQLViewTestSuite with SharedSparkSession {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org