This is an automated email from the ASF dual-hosted git repository.

cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 0917659a4695 [SPARK-60829][SQL] Register SQL configs into a 
ConcurrentHashMap instead of copy-on-write
0917659a4695 is described below

commit 0917659a4695d1bc0cf1a509bc7e3bed794898cf
Author: aaronkwan-db <[email protected]>
AuthorDate: Tue Nov 3 07:01:54 2026 +0000

    [SPARK-60829][SQL] Register SQL configs into a ConcurrentHashMap instead of 
copy-on-write
    
    ### What changes were proposed in this pull request?
    
    `SQLConf.sqlConfEntries` changes from a `volatile` copy-on-write 
`java.util.HashMap` to a `ConcurrentHashMap`:
    
    - `register` and `unregister` still hold `sqlConfEntriesUpdateLock`, keep 
the duplicate-key check, and still advance `sqlConfEntriesVersion` after the 
update. They now `put` or `remove` in place instead of copying the map.
    - `ConcurrentHashMap` rejects `null` keys, so a small `registeredEntry` 
helper keeps `getConfigEntry(null)` and `containsConfigKey(null)` returning 
`null` and `false`, as before.
    
    ### Why are the changes needed?
    
    Since SPARK-35701, every registration copies the whole registry, so 
registering n entries at class initialization does O(n^2) work. `SQLConf` alone 
registers ~900 entries, so its first access performs hundreds of thousands of 
entry copies plus repeated large `HashMap` allocations, all on the startup 
path. `ConcurrentHashMap` keeps the lock-free reads that motivated SPARK-35701 
and makes each registration O(1).
    
    Initialization of `object SQLConf` in a fresh JVM drops from 926 ms to 889 
ms (median of 15 interleaved runs, JDK 17).
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    - New tests in `SQLConfEntrySuite`:
      - Iterating `getConfigEntries()` while another thread registers entries. 
This guards against a non-concurrent map throwing 
`ConcurrentModificationException`; every entry registered before the iteration 
began must be seen.
      - Null-key lookups.
    - `build/mvn -pl sql/core -Dtest=none 
-DwildcardSuites=org.apache.spark.sql.internal.SQLConfEntrySuite,org.apache.spark.sql.internal.SQLConfSuite
 test` passes.
    - Micro-benchmark: timed 
`Class.forName("org.apache.spark.sql.internal.SQLConf$")` in fresh JVMs, 
alternating the patched and unpatched `spark-catalyst` jar for 15 runs each.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Code (Claude Opus 5.5)
    
    This pull request and its description were written by Isaac.
    
    Closes #60044 from aaronkwan-db/SPARK-sqlconf-concurrent-registry.
    
    Authored-by: aaronkwan-db <[email protected]>
    Signed-off-by: Wenchen Fan <[email protected]>
---
 .../org/apache/spark/sql/internal/SQLConf.scala    | 22 ++++-----
 .../spark/sql/internal/SQLConfEntrySuite.scala     | 56 ++++++++++++++++++++++
 2 files changed, 67 insertions(+), 11 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 1c8e339c032a..44293c2d0062 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.internal
 
 import java.util.{Locale, Properties, TimeZone}
 import java.util
-import java.util.concurrent.TimeUnit
+import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
 import java.util.concurrent.atomic.AtomicReference
 import java.util.regex.Pattern
 
@@ -66,8 +66,8 @@ object SQLConf {
 
   private[this] val sqlConfEntriesUpdateLock = new Object
 
-  @volatile
-  private[this] var sqlConfEntries: util.Map[String, ConfigEntry[_]] = 
util.Collections.emptyMap()
+  // Concurrent map so registration does not copy the whole registry and reads 
stay lock-free.
+  private[this] val sqlConfEntries = new ConcurrentHashMap[String, 
ConfigEntry[_]]()
 
   // Advanced after every `sqlConfEntries` update, so a reader that sees a 
version also sees the
   // entries it counts.
@@ -85,17 +85,13 @@ object SQLConf {
     require(!ConfigRegistry.containsConfig(entry.key),
       s"${entry.key} is defined in a .textproto config file. " +
         s"Use buildConfFromConfigFile instead of buildConf.")
-    val updatedMap = new java.util.HashMap[String, 
ConfigEntry[_]](sqlConfEntries)
-    updatedMap.put(entry.key, entry)
-    sqlConfEntries = updatedMap
+    sqlConfEntries.put(entry.key, entry)
     sqlConfEntriesVersion += 1
   }
 
   // For testing only
   private[sql] def unregister(entry: ConfigEntry[_]): Unit = 
sqlConfEntriesUpdateLock.synchronized {
-    val updatedMap = new java.util.HashMap[String, 
ConfigEntry[_]](sqlConfEntries)
-    updatedMap.remove(entry.key)
-    sqlConfEntries = updatedMap
+    sqlConfEntries.remove(entry.key)
     sqlConfEntriesVersion += 1
   }
 
@@ -134,8 +130,12 @@ object SQLConf {
       s"$settingsVersion, entriesVersion=$entriesVersion, 
keys=${retained.size})"
   }
 
+  // ConcurrentHashMap rejects null keys; treat a null key as unregistered.
+  private[this] def registeredEntry(key: String): ConfigEntry[_] =
+    if (key == null) null else sqlConfEntries.get(key)
+
   private[internal] def getConfigEntry(key: String): ConfigEntry[_] = {
-    
Option(sqlConfEntries.get(key)).getOrElse(ConfigEntry.findProtoDefinedEntry(key))
+    
Option(registeredEntry(key)).getOrElse(ConfigEntry.findProtoDefinedEntry(key))
   }
 
   private[sql] def isSessionBindingPolicy(key: String): Boolean = {
@@ -171,7 +171,7 @@ object SQLConf {
   }
 
   private[sql] def containsConfigKey(key: String): Boolean = {
-    sqlConfEntries.containsKey(key) || ConfigRegistry.containsConfig(key)
+    registeredEntry(key) != null || ConfigRegistry.containsConfig(key)
   }
 
   def registerStaticConfigKey(key: String): Unit = 
staticConfKeysUpdateLock.synchronized {
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala
index 9c34e28e5fe8..4a954eac8beb 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala
@@ -17,8 +17,19 @@
 
 package org.apache.spark.sql.internal
 
+import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit}
+import java.util.concurrent.atomic.AtomicBoolean
+
+import scala.collection.mutable
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
+import scala.jdk.CollectionConverters._
+import scala.util.Try
+
 import org.apache.spark.{SparkFunSuite, SparkIllegalArgumentException}
+import org.apache.spark.internal.config.ConfigEntry
 import org.apache.spark.sql.internal.SQLConf._
+import org.apache.spark.util.ThreadUtils
 
 class SQLConfEntrySuite extends SparkFunSuite {
 
@@ -253,4 +264,49 @@ class SQLConfEntrySuite extends SparkFunSuite {
     original.setConfString(key, "dontcopyme")
     assert(clone.getConfString(key, "noentry") === "clone")
   }
+
+  test("getConfigEntries can be iterated while entries are registered") {
+    // An overlapping iteration sees every entry registered before it began; 
later ones may vary.
+    // Guards against an in-place map throwing ConcurrentModificationException 
mid-iteration.
+    val prefix = "spark.sql.SQLConfEntrySuite.concurrent"
+    val beforeKeys = (0 until 50).map(i => s"$prefix.before$i")
+    val duringKeys = (0 until 200).map(i => s"$prefix.during$i")
+    // Every entry this test registers, so cleanup unregisters exactly those.
+    val registered = new ConcurrentLinkedQueue[ConfigEntry[_]]()
+    val stopWriter = new AtomicBoolean(false)
+    val iterating = new CountDownLatch(1)
+    val firstRegistered = new CountDownLatch(1)
+    var writer: Option[Future[Unit]] = None
+    try {
+      beforeKeys.foreach(key => 
registered.add(buildConf(key).intConf.createWithDefault(0)))
+      writer = Some(Future {
+        assert(iterating.await(1, TimeUnit.MINUTES))
+        duringKeys.iterator.takeWhile(_ => !stopWriter.get()).foreach { key =>
+          registered.add(buildConf(key).intConf.createWithDefault(0))
+          firstRegistered.countDown()
+        }
+      }(ExecutionContext.global))
+      val iterator = SQLConf.getConfigEntries().iterator()
+      val seen = mutable.HashSet(iterator.next().key)
+      iterating.countDown()
+      // At least one registration happens while this iteration is open.
+      assert(firstRegistered.await(1, TimeUnit.MINUTES))
+      while (iterator.hasNext) seen += iterator.next().key
+      beforeKeys.foreach(key => assert(seen.contains(key), key))
+      ThreadUtils.awaitResult(writer.get, 1.minute)
+      val after = SQLConf.getConfigEntries().asScala.map(_.key).toSet
+      (beforeKeys ++ duringKeys).foreach(key => assert(after.contains(key), 
key))
+    } finally {
+      stopWriter.set(true)
+      iterating.countDown()
+      // The writer stops at its next key, so this join is bounded; it must 
finish before cleanup.
+      writer.foreach(w => Try(ThreadUtils.awaitResult(w, Duration.Inf)))
+      registered.asScala.foreach(SQLConf.unregister)
+    }
+  }
+
+  test("registry lookups tolerate a null key") {
+    assert(SQLConf.getConfigEntry(null) === null)
+    assert(!SQLConf.containsConfigKey(null))
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to