This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1f23edfa84aa [SPARK-46827][CORE] Make `RocksDBPersistenceEngine` to
support a symbolic link
1f23edfa84aa is described below
commit 1f23edfa84aa3318791d5fbbbae22d479a49134a
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Wed Jan 24 07:35:14 2024 -0800
[SPARK-46827][CORE] Make `RocksDBPersistenceEngine` to support a symbolic
link
### What changes were proposed in this pull request?
This PR aims to make `RocksDBPersistenceEngine` to support a symbolic link
location.
### Why are the changes needed?
To be consistent with `FileSystemPersistenceEngine` which supports symbolic
link locations.
https://github.com/apache/spark/blob/7004dd9edcad32d34d0448df9498d32c444ab082/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala#L45-L50
### Does this PR introduce _any_ user-facing change?
No. This is a new feature at 4.0.0.
### How was this patch tested?
Pass the CIs with a newly added test case.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44867 from dongjoon-hyun/SPARK-46827.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../spark/deploy/master/RocksDBPersistenceEngine.scala | 9 +++++++--
.../spark/deploy/master/PersistenceEngineSuite.scala | 15 +++++++++++++++
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala
b/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala
index 5c43dab4d066..8364dbd693b1 100644
---
a/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.master
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets.UTF_8
-import java.nio.file.{Files, Paths}
+import java.nio.file.{FileAlreadyExistsException, Files, Paths}
import scala.collection.mutable.ArrayBuffer
import scala.reflect.ClassTag
@@ -43,7 +43,12 @@ private[master] class RocksDBPersistenceEngine(
RocksDB.loadLibrary()
- private val path = Files.createDirectories(Paths.get(dir))
+ private val path = try {
+ Files.createDirectories(Paths.get(dir))
+ } catch {
+ case _: FileAlreadyExistsException if Files.isSymbolicLink(Paths.get(dir))
=>
+ Files.createDirectories(Paths.get(dir).toRealPath())
+ }
/**
* Use full filter.
diff --git
a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
index b977a1142444..01b7e46eb2a8 100644
---
a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
@@ -88,6 +88,21 @@ class PersistenceEngineSuite extends SparkFunSuite {
}
}
+ test("SPARK-46827: RocksDBPersistenceEngine with a symbolic link") {
+ withTempDir { dir =>
+ val target = Paths.get(dir.getAbsolutePath(), "target")
+ val link = Paths.get(dir.getAbsolutePath(), "symbolic_link");
+
+ Files.createDirectories(target)
+ Files.createSymbolicLink(link, target);
+
+ val conf = new SparkConf()
+ testPersistenceEngine(conf, serializer =>
+ new RocksDBPersistenceEngine(link.toAbsolutePath.toString, serializer)
+ )
+ }
+ }
+
test("SPARK-46205: Support KryoSerializer in FileSystemPersistenceEngine") {
withTempDir { dir =>
val conf = new SparkConf()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
