This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new d1311f75dea [SPARK-46215][CORE] Improve `FileSystemPersistenceEngine` to allow nonexistent parents d1311f75dea is described below commit d1311f75dea21201f2c37b8f92b111c91ebc9aad Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Sat Dec 2 20:24:11 2023 -0800 [SPARK-46215][CORE] Improve `FileSystemPersistenceEngine` to allow nonexistent parents ### What changes were proposed in this pull request? This PR aims to improve `FileSystemPersistenceEngine` to allow non-exist parents ### Why are the changes needed? To prevent the following error, ``` java.io.IOException: No such file or directory at java.base/java.io.UnixFileSystem.createFileExclusively(Native Method) at java.base/java.io.File.createNewFile(File.java:1043) at org.apache.spark.deploy.master.FileSystemPersistenceEngine.serializeIntoFile(FileSystemPersistenceEngine.scala:62) at org.apache.spark.deploy.master.FileSystemPersistenceEngine.persist(FileSystemPersistenceEngine.scala:45) ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs with the newly added test case. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #44127 from dongjoon-hyun/SPARK-46215. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../apache/spark/deploy/master/FileSystemPersistenceEngine.scala | 5 +++-- .../org/apache/spark/deploy/master/PersistenceEngineSuite.scala | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala index 7f624816f25..094136ea274 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala @@ -18,6 +18,7 @@ package org.apache.spark.deploy.master import java.io._ +import java.nio.file.{Files, Paths} import scala.reflect.ClassTag @@ -31,7 +32,7 @@ import org.apache.spark.util.Utils * Stores data in a single on-disk directory with one file per application and worker. * Files are deleted when applications and workers are removed. * - * @param dir Directory to store files. Created if non-existent (but not recursively). + * @param dir Directory to store files. Created if non-existent. * @param serializer Used to serialize our objects. */ private[master] class FileSystemPersistenceEngine( @@ -39,7 +40,7 @@ private[master] class FileSystemPersistenceEngine( val serializer: Serializer) extends PersistenceEngine with Logging { - new File(dir).mkdir() + Files.createDirectories(Paths.get(dir)) override def persist(name: String, obj: Object): Unit = { serializeIntoFile(new File(dir + File.separator + name), obj) diff --git a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala index 66a61d80d2a..8546f4e01f3 100644 --- a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala @@ -53,6 +53,15 @@ class PersistenceEngineSuite extends SparkFunSuite { } } + test("SPARK-46215: FileSystemPersistenceEngine with a non-existent parent dir") { + withTempDir { dir => + val conf = new SparkConf() + testPersistenceEngine(conf, serializer => + new FileSystemPersistenceEngine(dir.getAbsolutePath + "/a/b/c/dir", serializer) + ) + } + } + test("SPARK-46205: Support KryoSerializer in FileSystemPersistenceEngine") { withTempDir { dir => val conf = new SparkConf() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org