This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 0917659a4695 [SPARK-60829][SQL] Register SQL configs into a
ConcurrentHashMap instead of copy-on-write
0917659a4695 is described below
commit 0917659a4695d1bc0cf1a509bc7e3bed794898cf
Author: aaronkwan-db <[email protected]>
AuthorDate: Tue Nov 3 07:01:54 2026 +0000
[SPARK-60829][SQL] Register SQL configs into a ConcurrentHashMap instead of
copy-on-write
### What changes were proposed in this pull request?
`SQLConf.sqlConfEntries` changes from a `volatile` copy-on-write
`java.util.HashMap` to a `ConcurrentHashMap`:
- `register` and `unregister` still hold `sqlConfEntriesUpdateLock`, keep
the duplicate-key check, and still advance `sqlConfEntriesVersion` after the
update. They now `put` or `remove` in place instead of copying the map.
- `ConcurrentHashMap` rejects `null` keys, so a small `registeredEntry`
helper keeps `getConfigEntry(null)` and `containsConfigKey(null)` returning
`null` and `false`, as before.
### Why are the changes needed?
Since SPARK-35701, every registration copies the whole registry, so
registering n entries at class initialization does O(n^2) work. `SQLConf` alone
registers ~900 entries, so its first access performs hundreds of thousands of
entry copies plus repeated large `HashMap` allocations, all on the startup
path. `ConcurrentHashMap` keeps the lock-free reads that motivated SPARK-35701
and makes each registration O(1).
Initialization of `object SQLConf` in a fresh JVM drops from 926 ms to 889
ms (median of 15 interleaved runs, JDK 17).
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- New tests in `SQLConfEntrySuite`:
- Iterating `getConfigEntries()` while another thread registers entries.
This guards against a non-concurrent map throwing
`ConcurrentModificationException`; every entry registered before the iteration
began must be seen.
- Null-key lookups.
- `build/mvn -pl sql/core -Dtest=none
-DwildcardSuites=org.apache.spark.sql.internal.SQLConfEntrySuite,org.apache.spark.sql.internal.SQLConfSuite
test` passes.
- Micro-benchmark: timed
`Class.forName("org.apache.spark.sql.internal.SQLConf$")` in fresh JVMs,
alternating the patched and unpatched `spark-catalyst` jar for 15 runs each.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5.5)
This pull request and its description were written by Isaac.
Closes #60044 from aaronkwan-db/SPARK-sqlconf-concurrent-registry.
Authored-by: aaronkwan-db <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../org/apache/spark/sql/internal/SQLConf.scala | 22 ++++-----
.../spark/sql/internal/SQLConfEntrySuite.scala | 56 ++++++++++++++++++++++
2 files changed, 67 insertions(+), 11 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 1c8e339c032a..44293c2d0062 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.internal
import java.util.{Locale, Properties, TimeZone}
import java.util
-import java.util.concurrent.TimeUnit
+import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
import java.util.concurrent.atomic.AtomicReference
import java.util.regex.Pattern
@@ -66,8 +66,8 @@ object SQLConf {
private[this] val sqlConfEntriesUpdateLock = new Object
- @volatile
- private[this] var sqlConfEntries: util.Map[String, ConfigEntry[_]] =
util.Collections.emptyMap()
+ // Concurrent map so registration does not copy the whole registry and reads
stay lock-free.
+ private[this] val sqlConfEntries = new ConcurrentHashMap[String,
ConfigEntry[_]]()
// Advanced after every `sqlConfEntries` update, so a reader that sees a
version also sees the
// entries it counts.
@@ -85,17 +85,13 @@ object SQLConf {
require(!ConfigRegistry.containsConfig(entry.key),
s"${entry.key} is defined in a .textproto config file. " +
s"Use buildConfFromConfigFile instead of buildConf.")
- val updatedMap = new java.util.HashMap[String,
ConfigEntry[_]](sqlConfEntries)
- updatedMap.put(entry.key, entry)
- sqlConfEntries = updatedMap
+ sqlConfEntries.put(entry.key, entry)
sqlConfEntriesVersion += 1
}
// For testing only
private[sql] def unregister(entry: ConfigEntry[_]): Unit =
sqlConfEntriesUpdateLock.synchronized {
- val updatedMap = new java.util.HashMap[String,
ConfigEntry[_]](sqlConfEntries)
- updatedMap.remove(entry.key)
- sqlConfEntries = updatedMap
+ sqlConfEntries.remove(entry.key)
sqlConfEntriesVersion += 1
}
@@ -134,8 +130,12 @@ object SQLConf {
s"$settingsVersion, entriesVersion=$entriesVersion,
keys=${retained.size})"
}
+ // ConcurrentHashMap rejects null keys; treat a null key as unregistered.
+ private[this] def registeredEntry(key: String): ConfigEntry[_] =
+ if (key == null) null else sqlConfEntries.get(key)
+
private[internal] def getConfigEntry(key: String): ConfigEntry[_] = {
-
Option(sqlConfEntries.get(key)).getOrElse(ConfigEntry.findProtoDefinedEntry(key))
+
Option(registeredEntry(key)).getOrElse(ConfigEntry.findProtoDefinedEntry(key))
}
private[sql] def isSessionBindingPolicy(key: String): Boolean = {
@@ -171,7 +171,7 @@ object SQLConf {
}
private[sql] def containsConfigKey(key: String): Boolean = {
- sqlConfEntries.containsKey(key) || ConfigRegistry.containsConfig(key)
+ registeredEntry(key) != null || ConfigRegistry.containsConfig(key)
}
def registerStaticConfigKey(key: String): Unit =
staticConfKeysUpdateLock.synchronized {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala
index 9c34e28e5fe8..4a954eac8beb 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfEntrySuite.scala
@@ -17,8 +17,19 @@
package org.apache.spark.sql.internal
+import java.util.concurrent.{ConcurrentLinkedQueue, CountDownLatch, TimeUnit}
+import java.util.concurrent.atomic.AtomicBoolean
+
+import scala.collection.mutable
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
+import scala.jdk.CollectionConverters._
+import scala.util.Try
+
import org.apache.spark.{SparkFunSuite, SparkIllegalArgumentException}
+import org.apache.spark.internal.config.ConfigEntry
import org.apache.spark.sql.internal.SQLConf._
+import org.apache.spark.util.ThreadUtils
class SQLConfEntrySuite extends SparkFunSuite {
@@ -253,4 +264,49 @@ class SQLConfEntrySuite extends SparkFunSuite {
original.setConfString(key, "dontcopyme")
assert(clone.getConfString(key, "noentry") === "clone")
}
+
+ test("getConfigEntries can be iterated while entries are registered") {
+ // An overlapping iteration sees every entry registered before it began;
later ones may vary.
+ // Guards against an in-place map throwing ConcurrentModificationException
mid-iteration.
+ val prefix = "spark.sql.SQLConfEntrySuite.concurrent"
+ val beforeKeys = (0 until 50).map(i => s"$prefix.before$i")
+ val duringKeys = (0 until 200).map(i => s"$prefix.during$i")
+ // Every entry this test registers, so cleanup unregisters exactly those.
+ val registered = new ConcurrentLinkedQueue[ConfigEntry[_]]()
+ val stopWriter = new AtomicBoolean(false)
+ val iterating = new CountDownLatch(1)
+ val firstRegistered = new CountDownLatch(1)
+ var writer: Option[Future[Unit]] = None
+ try {
+ beforeKeys.foreach(key =>
registered.add(buildConf(key).intConf.createWithDefault(0)))
+ writer = Some(Future {
+ assert(iterating.await(1, TimeUnit.MINUTES))
+ duringKeys.iterator.takeWhile(_ => !stopWriter.get()).foreach { key =>
+ registered.add(buildConf(key).intConf.createWithDefault(0))
+ firstRegistered.countDown()
+ }
+ }(ExecutionContext.global))
+ val iterator = SQLConf.getConfigEntries().iterator()
+ val seen = mutable.HashSet(iterator.next().key)
+ iterating.countDown()
+ // At least one registration happens while this iteration is open.
+ assert(firstRegistered.await(1, TimeUnit.MINUTES))
+ while (iterator.hasNext) seen += iterator.next().key
+ beforeKeys.foreach(key => assert(seen.contains(key), key))
+ ThreadUtils.awaitResult(writer.get, 1.minute)
+ val after = SQLConf.getConfigEntries().asScala.map(_.key).toSet
+ (beforeKeys ++ duringKeys).foreach(key => assert(after.contains(key),
key))
+ } finally {
+ stopWriter.set(true)
+ iterating.countDown()
+ // The writer stops at its next key, so this join is bounded; it must
finish before cleanup.
+ writer.foreach(w => Try(ThreadUtils.awaitResult(w, Duration.Inf)))
+ registered.asScala.foreach(SQLConf.unregister)
+ }
+ }
+
+ test("registry lookups tolerate a null key") {
+ assert(SQLConf.getConfigEntry(null) === null)
+ assert(!SQLConf.containsConfigKey(null))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]