maheshk114 commented on code in PR #52507:
URL: https://github.com/apache/spark/pull/52507#discussion_r2486193408
##########
core/src/main/scala/org/apache/spark/internal/io/HadoopMapReduceCommitProtocol.scala:
##########
@@ -197,33 +201,66 @@ class HadoopMapReduceCommitProtocol(
}
logDebug(s"Create absolute parent directories: $absParentPaths")
absParentPaths.foreach(fs.mkdirs)
- for ((src, dst) <- filesToMove) {
- if (!fs.rename(new Path(src), new Path(dst))) {
- throw new IOException(s"Failed to rename $src to $dst when
committing files staged for " +
- s"absolute locations")
+
+ val numThreads = SparkContext.getActive match {
Review Comment:
use SparkEnv ..it is available in both Driver and Executor.
##########
sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala:
##########
@@ -122,13 +126,25 @@ case class InsertIntoHiveDirCommand(
}
val dfs = tmpPath.getFileSystem(hadoopConf)
- dfs.listStatus(tmpPath).foreach {
- tmpFile =>
- if (isLocal) {
- dfs.copyToLocalFile(tmpFile.getPath, writeToPath)
- } else {
- dfs.rename(tmpFile.getPath, writeToPath)
- }
+ val tmpFiles = dfs.listStatus(tmpPath)
+
+ val numThreads =
sparkSession.sparkContext.conf.get(FILES_RENAME_NUM_THREADS)
+ val pool = ThreadUtils.newDaemonFixedThreadPool(numThreads,
"hive-dir-rename")
Review Comment:
why the thread pool is created per job here ?
##########
core/src/main/scala/org/apache/spark/internal/config/package.scala:
##########
@@ -1240,6 +1240,13 @@ package object config {
.booleanConf
.createWithDefault(false)
+ private[spark] val FILES_RENAME_NUM_THREADS =
ConfigBuilder("spark.files.rename.numThreads")
Review Comment:
If per session then you need to have per job thread pool.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
