squito commented on a change in pull request #23393: [SPARK-26288][CORE]add
initRegisteredExecutorsDB
URL: https://github.com/apache/spark/pull/23393#discussion_r245364298
##########
File path:
core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
##########
@@ -56,11 +60,55 @@ class ExternalShuffleService(sparkConf: SparkConf,
securityManager: SecurityMana
private var server: TransportServer = _
+ private final val MAX_DIR_CREATION_ATTEMPTS = 10
+
private val shuffleServiceSource = new ExternalShuffleServiceSource
+ protected def createDirectory(root: String, name: String): File = {
+ var attempts = 0
+ val maxAttempts = MAX_DIR_CREATION_ATTEMPTS
+ var dir: File = null
+ while (dir == null) {
+ attempts += 1
+ if (attempts > maxAttempts) {
+ throw new IOException("Failed to create a temp directory (under " +
root + ") after " +
+ maxAttempts + " attempts!")
+ }
+ try {
+ dir = new File(root, "registeredExecutors")
+ if (!dir.exists() && !dir.mkdirs()) {
+ dir = null
+ }
+ } catch { case e: SecurityException => dir = null; }
+ }
+ logInfo(s"registeredExecutorsDb path is ${dir.getAbsolutePath}")
+ new File(dir.getAbsolutePath, name)
+ }
+
+ protected def initRegisteredExecutorsDB(dbName: String): File = {
+ val localDirs = sparkConf.get("spark.local.dir", "").split(",")
+ if (localDirs.length >= 1 && !"".equals(localDirs(0))) {
+ createDirectory(localDirs(0), dbName)
Review comment:
any reason to only use `localDirs(0)` instead of checking all localDirs? I
worry that you might get another local dir pre-pended or something during a
configuration change + restart
also it seems you're creating a path like
"[local-dir]/registeredExecutors/registeredExecutors.ldb", any reason for the
extra level?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
