Github user edwinalu commented on a diff in the pull request:
https://github.com/apache/spark/pull/21221#discussion_r190995781
--- Diff: core/src/main/scala/org/apache/spark/executor/Executor.scala ---
@@ -800,26 +812,50 @@ private[spark] class Executor(
}
}
}
-
- /**
- * Schedules a task to report heartbeat and partial metrics for active
tasks to driver.
- */
- private def startDriverHeartbeater(): Unit = {
- val intervalMs = conf.getTimeAsMs("spark.executor.heartbeatInterval",
"10s")
-
- // Wait a random interval so the heartbeats don't end up in sync
- val initialDelay = intervalMs + (math.random *
intervalMs).asInstanceOf[Int]
-
- val heartbeatTask = new Runnable() {
- override def run(): Unit =
Utils.logUncaughtExceptions(reportHeartBeat())
- }
- heartbeater.scheduleAtFixedRate(heartbeatTask, initialDelay,
intervalMs, TimeUnit.MILLISECONDS)
- }
}
private[spark] object Executor {
// This is reserved for internal use by components that need to read
task properties before a
// task is fully deserialized. When possible, the
TaskContext.getLocalProperty call should be
// used instead.
val taskDeserializationProps: ThreadLocal[Properties] = new
ThreadLocal[Properties]
+
+ val DIRECT_BUFFER_POOL_NAME = "direct"
+ val MAPPED_BUFFER_POOL_NAME = "mapped"
+
+ /** Get the BufferPoolMXBean for the specified buffer pool. */
+ def getBufferPool(pool: String): BufferPoolMXBean = {
+ val name = new ObjectName("java.nio:type=BufferPool,name=" + pool)
+
ManagementFactory.newPlatformMXBeanProxy(ManagementFactory.getPlatformMBeanServer,
+ name.toString, classOf[BufferPoolMXBean])
+ }
+
+ /**
+ * Get the current executor level memory metrics.
+ *
+ * @param memoryManager the memory manager
+ * @param direct the direct memory buffer pool
+ * @param mapped the mapped memory buffer pool
+ * @return the executor memory metrics
+ */
+ def getCurrentExecutorMetrics(
+ memoryManager: MemoryManager,
+ direct: BufferPoolMXBean,
+ mapped: BufferPoolMXBean) : ExecutorMetrics = {
--- End diff --
Yes, and easier to share the code between driver and executor.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
