andygrove commented on code in PR #1702:
URL: https://github.com/apache/datafusion-comet/pull/1702#discussion_r2072417946
##########
native/core/src/execution/jni_api.rs:
##########
@@ -359,6 +367,41 @@ pub unsafe extern "system" fn
Java_org_apache_comet_Native_executePlan(
// Retrieve the query
let exec_context = get_execution_context(exec_context);
+ // memory profiling is only available on linux
+ if exec_context.memory_profiling_enabled {
+ #[cfg(target_os = "linux")]
+ {
+ let pid = std::process::id();
+ let process = Process::new(pid as i32).unwrap();
+ let statm = process.statm().unwrap();
+ let page_size = procfs::page_size();
+ println!(
+ "NATIVE_MEMORY: {{ resident: {:.0} }}",
+ (statm.resident * page_size) as f64 / (1024.0 * 1024.0)
+ );
+
+ #[cfg(feature = "jemalloc")]
+ {
+ // Obtain a MIB for the `epoch`, `stats.allocated`, and
+ // `atats.resident` keys:
+ let e = epoch::mib().unwrap();
+ let allocated = stats::allocated::mib().unwrap();
+ let resident = stats::resident::mib().unwrap();
+
+ // Many statistics are cached and only updated
+ // when the epoch is advanced:
+ e.advance().unwrap();
+
+ // Read statistics using MIB key:
+ let allocated = allocated.read().unwrap() as f64 / (1024.0
* 1024.0);
+ let resident = resident.read().unwrap() as f64 / (1024.0 *
1024.0);
+ println!(
Review Comment:
These metrics are for the overall process, so not specific to any particular
plan. This feature just allows us to watch the executor logs and see when
memory starts to approach the limit. I hope this will help us debug the cause
of OOMs, although it will still primarily be a manual process.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
