This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 1030678a97 [GLUTEN-10707] Improve SparkDirectoryUtil by reduce
synchronized (#10708)
1030678a97 is described below
commit 1030678a97acb10e88ccd99257dc88d0d28126f1
Author: Jiaan Geng <[email protected]>
AuthorDate: Mon Oct 13 15:27:14 2025 +0800
[GLUTEN-10707] Improve SparkDirectoryUtil by reduce synchronized (#10708)
---
.../org/apache/spark/util/SparkDirectoryUtil.scala | 25 +++++++++++-----------
1 file changed, 13 insertions(+), 12 deletions(-)
diff --git
a/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
index 24379853d3..8e62ad7ae7 100644
--- a/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala
@@ -62,30 +62,31 @@ class SparkDirectoryUtil private (val roots: Array[String])
extends Logging {
}
object SparkDirectoryUtil extends Logging {
- private var INSTANCE: SparkDirectoryUtil = _
+ @volatile private var roots: Array[String] = _
+ private lazy val INSTANCE: SparkDirectoryUtil = {
+ if (this.roots == null) {
+ throw new IllegalStateException("SparkDirectoryUtil not initialized")
+ }
+ new SparkDirectoryUtil(this.roots)
+ }
- def init(conf: SparkConf): Unit = synchronized {
+ def init(conf: SparkConf): Unit = {
val roots = Utils.getConfiguredLocalDirs(conf)
init(roots)
}
private def init(roots: Array[String]): Unit = synchronized {
- if (INSTANCE == null) {
- INSTANCE = new SparkDirectoryUtil(roots)
- return
- }
- if (INSTANCE.roots.toSet != roots.toSet) {
+ if (this.roots == null) {
+ this.roots = roots
+ } else if (this.roots.toSet != roots.toSet) {
throw new IllegalArgumentException(
- s"Reinitialize SparkDirectoryUtil with different root dirs: old:
${INSTANCE.ROOTS
+ s"Reinitialize SparkDirectoryUtil with different root dirs: old:
${this.roots
.mkString("Array(", ", ", ")")}, new: ${roots.mkString("Array(",
", ", ")")}"
)
}
}
- def get(): SparkDirectoryUtil = synchronized {
- assert(INSTANCE != null, "Default instance of SparkDirectoryUtil was not
set yet")
- INSTANCE
- }
+ def get(): SparkDirectoryUtil = INSTANCE
}
class Namespace(private val parents: Array[File], private val name: String) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
