This is an automated email from the ASF dual-hosted git repository.
dtenedor pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 730e4306d6ca [SPARK-59997][SQL] Avoid full config registry scans
during body resolution
730e4306d6ca is described below
commit 730e4306d6ca06f2a10954eda9af7de324573350
Author: Vladimir Golubev <[email protected]>
AuthorDate: Tue Oct 6 10:37:35 2026 -0700
[SPARK-59997][SQL] Avoid full config registry scans during body resolution
### What changes were proposed in this pull request?
1. Goal: Reduce configuration overhead when resolving views and SQL
function bodies.
2. Problem: Each call scans `SQLConf.getConfigEntries()` and builds a
retained-key set,
even when the session has only a few explicitly set configurations.
3. Solution: Classify explicitly set keys with live policy lookups,
preserving captured
values, catalog configurations, and the existing ANSI fallback.
### Why are the changes needed?
Enumerating the full registry and allocating a retained-key set is
unnecessary when
only explicitly set session configurations can be copied.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New tests.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex 0.156.1
Closes #59254 from vladimirg-db/optimize-resolution-config-retention.
Authored-by: Vladimir Golubev <[email protected]>
Signed-off-by: Daniel Tenedorio <[email protected]>
(cherry picked from commit e10ed1293151d3a347e31f48a620d4bd7fa9f361)
Signed-off-by: Daniel Tenedorio <[email protected]>
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 21 ++-------
.../org/apache/spark/sql/internal/SQLConf.scala | 7 +++
.../sql/analysis/AnalysisConfOverrideSuite.scala | 54 ++++++++++++++++++++++
3 files changed, 66 insertions(+), 16 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 85e4a5e07368..a1d698ed50f1 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -26,7 +26,6 @@ import scala.jdk.CollectionConverters._
import scala.util.{Failure, Random, Success, Try}
import org.apache.spark.{SparkException, SparkUnsupportedOperationException}
-import org.apache.spark.internal.config.ConfigBindingPolicy
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst._
import
org.apache.spark.sql.catalyst.analysis.TableOutputResolver.DefaultValueFillMode._
@@ -322,25 +321,15 @@ object AnalysisContext {
object Analyzer {
// Configs with bindingPolicy SESSION or NOT_APPLICABLE are retained when
resolving views and
// SQL UDFs, so that their values propagate from the active session rather
than falling back to
- // Spark defaults. Note: configs defined in lazily-loaded modules (e.g.,
sql/hive) will only
- // be included if their holding Scala object has been initialized before
this set is computed.
+ // Spark defaults. Entries from lazily-loaded modules are recognized once
registered.
def retainResolutionConfigsForAnalysis(
newConf: SQLConf,
existingConf: SQLConf,
createSparkVersion: String = ""): Unit = {
- val retainedConfigKeys = SQLConf.getConfigEntries().asScala
- .filter(entry =>
- entry.bindingPolicy.contains(ConfigBindingPolicy.SESSION) ||
- entry.bindingPolicy.contains(ConfigBindingPolicy.NOT_APPLICABLE))
- .map(_.key)
- .toSet
-
- val retainedConfigs = existingConf.getAllConfs.filter { case (key, _) =>
- retainedConfigKeys.contains(key) || key.startsWith("spark.sql.catalog.")
- }
-
- retainedConfigs.foreach { case (k, v) =>
- newConf.settings.put(k, v)
+ existingConf.getAllConfs.foreach { case (key, value) =>
+ if (key.startsWith("spark.sql.catalog.") ||
SQLConf.isSessionBindingPolicy(key)) {
+ newConf.settings.put(key, value)
+ }
}
trySetAnsiValue(newConf, createSparkVersion)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 5ec5479683b0..898c9023e971 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -96,6 +96,13 @@ object SQLConf {
Option(sqlConfEntries.get(key)).getOrElse(ConfigEntry.findProtoDefinedEntry(key))
}
+ private[sql] def isSessionBindingPolicy(key: String): Boolean = {
+ Option(getConfigEntry(key)).exists { entry =>
+ entry.bindingPolicy.contains(ConfigBindingPolicy.SESSION) ||
+ entry.bindingPolicy.contains(ConfigBindingPolicy.NOT_APPLICABLE)
+ }
+ }
+
// TODO: once all configs are migrated to textproto, this can be replaced by
// ConfigEntry.listAllEntries() and callers can filter by config properties.
private[sql] def getConfigEntries(): util.Collection[ConfigEntry[_]] = {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/analysis/AnalysisConfOverrideSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/analysis/AnalysisConfOverrideSuite.scala
index a473bb6a02d7..9317babc4650 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/analysis/AnalysisConfOverrideSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/analysis/AnalysisConfOverrideSuite.scala
@@ -19,9 +19,12 @@ package org.apache.spark.sql.analysis
import org.apache.spark.SparkConf
import org.apache.spark.SparkNoSuchElementException
+import org.apache.spark.internal.config.ConfigBindingPolicy
import org.apache.spark.sql.SparkSessionExtensions
+import org.apache.spark.sql.catalyst.analysis.Analyzer
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
class AnalysisConfOverrideSuite extends SharedSparkSession {
@@ -53,6 +56,57 @@ class AnalysisConfOverrideSuite extends SharedSparkSession {
}
}
+ test("resolution config retention follows binding policies") {
+ val sessionKey = SQLConf.ALLOW_CREATING_UDT_FROM_STRING.key
+ val notApplicableKey = SQLConf.SPLIT_PROJECTION_FOR_EXPENSIVE_FILTERS.key
+ val persistedKey = SQLConf.CHAR_VARCHAR_STANDARD_SEMANTICS.key
+ val catalogKey = "spark.sql.catalog.analysisConfOverrideSuite.option"
+ val unknownKey = "spark.sql.analysisConfOverrideSuite.unknown." +
+ java.util.UUID.randomUUID().toString.replace("-", "")
+
+ val existingConf = new SQLConf()
+ existingConf.setConfString(sessionKey, "false")
+ existingConf.setConfString(notApplicableKey, "false")
+ existingConf.setConfString(persistedKey, "false")
+ existingConf.setConfString(catalogKey, "session-catalog")
+ existingConf.setConfString(unknownKey, "session")
+
+ val newConf = new SQLConf()
+ newConf.setConfString(sessionKey, "true")
+ newConf.setConfString(notApplicableKey, "true")
+ newConf.setConfString(persistedKey, "true")
+ newConf.setConfString(catalogKey, "captured-catalog")
+ newConf.setConfString(unknownKey, "captured")
+
+ withSQLConf(SQLConf.ASSUME_ANSI_FALSE_IF_NOT_PERSISTED.key -> "true") {
+ Analyzer.retainResolutionConfigsForAnalysis(
+ newConf, existingConf, createSparkVersion = "4.0.0")
+ }
+
+ assert(newConf.getConfString(sessionKey) == "false")
+ assert(newConf.getConfString(notApplicableKey) == "false")
+ assert(newConf.getConfString(persistedKey) == "true")
+ assert(newConf.getConfString(catalogKey) == "session-catalog")
+ assert(newConf.getConfString(unknownKey) == "captured")
+ assert(newConf.getConfString(SQLConf.ANSI_ENABLED.key) == "true")
+
+ val dynamicEntry = SQLConf.buildConf(unknownKey)
+ .withBindingPolicy(ConfigBindingPolicy.SESSION)
+ .stringConf
+ .createWithDefault("default")
+ try {
+ Analyzer.retainResolutionConfigsForAnalysis(newConf, existingConf)
+ assert(newConf.getConfString(unknownKey) == "session")
+
+ SQLConf.unregister(dynamicEntry)
+ newConf.setConfString(unknownKey, "captured after unregistration")
+ Analyzer.retainResolutionConfigsForAnalysis(newConf, existingConf)
+ assert(newConf.getConfString(unknownKey) == "captured after
unregistration")
+ } finally {
+ SQLConf.unregister(dynamicEntry)
+ }
+ }
+
testOverride("simple plan") { case (key, value) =>
ValidateConfOverrideRule.withConfValidationEnabled(key, value) {
spark.sql("SELECT * FROM TaBlE")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
