This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b73b1dc [SPARK-34613][SQL] Fix view does not capture disable hint
config
b73b1dc is described below
commit b73b1dca54ec95d2085eb6d5daf659d695fea1a5
Author: ulysses-you <[email protected]>
AuthorDate: Fri Mar 5 12:19:30 2021 +0800
[SPARK-34613][SQL] Fix view does not capture disable hint config
### What changes were proposed in this pull request?
Add allow list to capture sql config for view.
### Why are the changes needed?
Spark use origin text sql to store view then capture and store sql config
into view metadata.
Capture config will skip some config with some prefix, e.g.
`spark.sql.optimizer.` but unfortunately `spark.sql.optimizer.disableHints` is
start with `spark.sql.optimizer.`.
We need a allow list to help capture the config.
### Does this PR introduce _any_ user-facing change?
Yes bug fix.
### How was this patch tested?
Add test.
Closes #31732 from ulysses-you/SPARK-34613.
Authored-by: ulysses-you <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 43aacd5069294e1215e86cd43bd0810bda998be2)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../org/apache/spark/sql/execution/command/views.scala | 12 +++++++++++-
.../org/apache/spark/sql/execution/SQLViewTestSuite.scala | 15 +++++++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 960fe4a..ef0e90d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -353,8 +353,18 @@ object ViewHelper {
"spark.sql.shuffle.",
"spark.sql.adaptive.")
+ private val configAllowList = Seq(
+ SQLConf.DISABLE_HINTS.key
+ )
+
+ /**
+ * Capture view config either of:
+ * 1. exists in allowList
+ * 2. do not exists in denyList
+ */
private def shouldCaptureConfig(key: String): Boolean = {
- !configPrefixDenyList.exists(prefix => key.startsWith(prefix))
+ configAllowList.exists(prefix => key.equals(prefix)) ||
+ !configPrefixDenyList.exists(prefix => key.startsWith(prefix))
}
import CatalogTable._
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
index 84a20bb..88218b1 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.execution
import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.plans.logical.Repartition
import org.apache.spark.sql.internal.SQLConf._
import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
@@ -278,6 +279,20 @@ abstract class SQLViewTestSuite extends QueryTest with
SQLTestUtils {
}
}
}
+
+ test("SPARK-34613: Fix view does not capture disable hint config") {
+ withSQLConf(DISABLE_HINTS.key -> "true") {
+ val viewName = createView("v1", "SELECT /*+ repartition(1) */ 1")
+ withView(viewName) {
+ assert(
+ sql(s"SELECT * FROM $viewName").queryExecution.analyzed.collect {
+ case e: Repartition => e
+ }.isEmpty
+ )
+ checkViewOutput(viewName, Seq(Row(1)))
+ }
+ }
+ }
}
class LocalTempViewTestSuite extends SQLViewTestSuite with SharedSparkSession {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
