This is an automated email from the ASF dual-hosted git repository.
kejia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 9cceba681 [CORE] Fix the java.nio.file.NoSuchFileException: default in
spark 3.5 (#6175)
9cceba681 is described below
commit 9cceba6812d4de9883695f20bc506648083715f9
Author: Xiduo You <[email protected]>
AuthorDate: Mon Jun 24 09:07:09 2024 +0800
[CORE] Fix the java.nio.file.NoSuchFileException: default in spark 3.5
(#6175)
---
gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala
b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala
index 9756837d9..04272517e 100644
--- a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala
@@ -41,8 +41,8 @@ object HdfsConfGenerator extends Logging {
addFileMethod.invoke(sc, path, Boolean.box(false), Boolean.box(true),
Boolean.box(false))
// Overwrite the spark internal config `spark.app.initial.file.urls`,
// so that the file can be available before initializing executor plugin.
- assert(sc.addedFiles.nonEmpty)
- sc.conf.set("spark.app.initial.file.urls",
sc.addedFiles.keys.toSeq.mkString(","))
+ assert(sc.listFiles.nonEmpty)
+ sc.conf.set("spark.app.initial.file.urls", sc.listFiles().mkString(","))
}
private def ignoreKey(key: String): Boolean = {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
