pan3793 commented on code in PR #49440:
URL: https://github.com/apache/spark/pull/49440#discussion_r1910118560
##########
connector/profiler/src/main/scala/org/apache/spark/executor/profiler/ExecutorJVMProfiler.scala:
##########
@@ -38,15 +38,26 @@ private[spark] class ExecutorJVMProfiler(conf: SparkConf,
executorId: String) ex
private var running = false
private val enableProfiler = conf.get(EXECUTOR_PROFILING_ENABLED)
private val profilerOptions = conf.get(EXECUTOR_PROFILING_OPTIONS)
- private val profilerDfsDir = conf.get(EXECUTOR_PROFILING_DFS_DIR)
+ private val profilerDfsDirOpt = conf.get(EXECUTOR_PROFILING_DFS_DIR)
private val profilerLocalDir = conf.get(EXECUTOR_PROFILING_LOCAL_DIR)
private val writeInterval = conf.get(EXECUTOR_PROFILING_WRITE_INTERVAL)
- private val startcmd =
s"start,$profilerOptions,file=$profilerLocalDir/profile.jfr"
- private val stopcmd =
s"stop,$profilerOptions,file=$profilerLocalDir/profile.jfr"
- private val dumpcmd =
s"dump,$profilerOptions,file=$profilerLocalDir/profile.jfr"
- private val resumecmd =
s"resume,$profilerOptions,file=$profilerLocalDir/profile.jfr"
+ private val appId = try {
+ conf.getAppId
+ } catch {
+ case _: NoSuchElementException => "local-" + System.currentTimeMillis
+ }
+ private val appAttemptId = conf.getOption("spark.app.attempt.id")
+ private val baseName = Utils.nameForAppAndAttempt(appId, appAttemptId)
+ private val profileFile = s"profile-$baseName-exec-$executorId.jfr"
+
+ private val startcmd =
s"start,$profilerOptions,file=$profilerLocalDir/$profileFile"
+ private val stopcmd =
s"stop,$profilerOptions,file=$profilerLocalDir/$profileFile"
+ private val dumpcmd =
s"dump,$profilerOptions,file=$profilerLocalDir/$profileFile"
+ private val resumecmd =
s"resume,$profilerOptions,file=$profilerLocalDir/$profileFile"
+ private val PROFILER_FOLDER_PERMISSIONS = new
FsPermission(Integer.parseInt("770", 8).toShort)
+ private val PROFILER_FILE_PERMISSIONS = new
FsPermission(Integer.parseInt("660", 8).toShort)
Review Comment:
follow event log behavior, to allow SHS process reading the file. we may do
the following things in the future:
- support downloading profiling results from SHS, like we have done for
event logs
- support integration with History UI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
