vanzin commented on a change in pull request #24901: [SPARK-28091[CORE] Extend
Spark metrics system with user-defined metrics using executor plugins
URL: https://github.com/apache/spark/pull/24901#discussion_r321902514
##########
File path: core/src/main/scala/org/apache/spark/executor/Executor.scala
##########
@@ -139,20 +139,28 @@ private[spark] class Executor(
private val executorPlugins: Seq[ExecutorPlugin] = {
val pluginNames = conf.get(EXECUTOR_PLUGINS)
if (pluginNames.nonEmpty) {
- logDebug(s"Initializing the following plugins: ${pluginNames.mkString(",
")}")
+ logDebug("Initializing the following executor plugin(s): " +
+ s"${pluginNames.mkString(", ")}")
+ val pluginSource = new ExecutorPluginSource
+ val pluginContext = new
ExecutorPluginContext(pluginSource.metricRegistry, conf, executorId)
// Plugins need to load using a class loader that includes the
executor's user classpath
val pluginList: Seq[ExecutorPlugin] =
Utils.withContextClassLoader(replClassLoader) {
- val plugins = Utils.loadExtensions(classOf[ExecutorPlugin],
pluginNames, conf)
+ val plugins = Utils.loadExtensions(
+ classOf[ExecutorPlugin], pluginNames, conf)
plugins.foreach { plugin =>
- plugin.init()
- logDebug(s"Successfully loaded plugin " +
plugin.getClass().getCanonicalName())
+ plugin.init(pluginContext)
+ logDebug(s"Successfully loaded the executor plugin " +
+ plugin.getClass().getCanonicalName())
}
plugins
}
- logDebug("Finished initializing plugins")
+ if (pluginSource.metricRegistry.getNames.size() > 0) {
+ env.metricsSystem.registerSource(pluginSource)
+ }
+ logDebug("Finished initializing executor plugins")
Review comment:
Same comment.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
