sandugood commented on code in PR #1547:
URL:
https://github.com/apache/datafusion-ballista/pull/1547#discussion_r3060836260
##########
ballista/executor/src/executor_server.rs:
##########
@@ -476,12 +497,44 @@ impl<T: 'static + AsLogicalPlan, U: 'static +
AsExecutionPlan> ExecutorServer<T,
};
}
- // TODO populate with real metrics
+ // Getting system-wide executor metrics
fn get_executor_metrics(&self) -> Vec<ExecutorMetric> {
+ let mut executor_system = self.executor_system.lock().unwrap();
+ let should_refresh = {
+ if executor_system.last_refresh.elapsed() >=
Duration::from_millis(100) {
+ executor_system.last_refresh = Instant::now();
+ true
+ } else {
+ false
+ }
+ };
+
+ if should_refresh {
+ executor_system.system.refresh_all();
+ }
+
+ let mut executor_metrics = Vec::new();
+
+ let total_memory = ExecutorMetric {
+ metric: Some(executor_metric::Metric::TotalMemory(
+ executor_system.system.total_memory(),
+ )),
+ };
+ executor_metrics.push(total_memory);
+
let available_memory = ExecutorMetric {
- metric: Some(executor_metric::Metric::AvailableMemory(u64::MAX)),
+ metric: Some(executor_metric::Metric::AvailableMemory(
+ executor_system.system.available_memory(),
+ )),
+ };
+ executor_metrics.push(available_memory);
+
+ let used_memory = ExecutorMetric {
+ metric: Some(executor_metric::Metric::UsedMemory(
+ executor_system.system.used_memory(),
+ )),
};
- let executor_metrics = vec![available_memory];
+ executor_metrics.push(used_memory);
Review Comment:
With newest commit it doesnt matter. This function is being called when we
send hearbeat to the scheduler, so as a "metric collector" tool we shouldnt
cache it I think and let it be recalculated at each heartbeat
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
