This is an automated email from the ASF dual-hosted git repository.
mridulm80 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new cf097a5ab7fc [SPARK-52776][CORE] Do not split the comm field in
ProcfsMetricsGetter
cf097a5ab7fc is described below
commit cf097a5ab7fcffcb64a765db9b8913304340506f
Author: Maxime Xu <ma...@linkedin.com>
AuthorDate: Mon Jul 14 11:54:10 2025 -0500
[SPARK-52776][CORE] Do not split the comm field in ProcfsMetricsGetter
### What changes were proposed in this pull request?
We are fixing an issue in `ProcfsMetricsGetter` when parsing the
`/proc/<pid>/stat` file. The current implementation will split the comm field
by spaces if it contains them, thereby causing subsequent numbers to be
shifted. The comm field, and only the comm field, is in parentheses so we can
resolve this issue by ignoring everything between the first open parenthesis
and last closing parenthesis when splitting the stat file.
### Why are the changes needed?
These changes are needed to prevent a comm field with spaces from causing
incorrect calculations for vmem/rssmem metrics. Please see
[JIRA](https://issues.apache.org/jira/projects/SPARK/issues/SPARK-52776) for
details.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Added a unit test to test for irregular characters in the comm field
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #51457 from max2718281/procfs.
Authored-by: Maxime Xu <ma...@linkedin.com>
Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com>
---
.../org/apache/spark/executor/ProcfsMetricsGetter.scala | 9 ++++++++-
core/src/test/resources/ProcfsMetrics/487713/stat | 1 +
.../org/apache/spark/executor/ProcfsMetricsGetterSuite.scala | 12 ++++++++++++
3 files changed, 21 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
index b9a462d62e41..00334e9cc55f 100644
--- a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
@@ -96,7 +96,14 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String =
"/proc/") extends L
}
Utils.tryWithResource(openReader()) { in =>
val procInfo = in.readLine
- val procInfoSplit = procInfo.split(" ")
+ // The comm field, which is inside parentheses, could contain spaces.
We should not split
+ // by those spaces as doing so could cause the numbers after it to be
shifted.
+ val commStartIndex = procInfo.indexOf('(')
+ val commEndIndex = procInfo.lastIndexOf(')') + 1
+ val pidArray = Array(procInfo.substring(0, commStartIndex).trim)
+ val commArray = Array(procInfo.substring(commStartIndex, commEndIndex))
+ val splitAfterComm = procInfo.substring(commEndIndex).trim.split(" ")
+ val procInfoSplit = pidArray ++ commArray ++ splitAfterComm
val vmem = procInfoSplit(22).toLong
val rssMem = procInfoSplit(23).toLong * pageSize
if (procInfoSplit(1).toLowerCase(Locale.US).contains("java")) {
diff --git a/core/src/test/resources/ProcfsMetrics/487713/stat
b/core/src/test/resources/ProcfsMetrics/487713/stat
new file mode 100644
index 000000000000..63640b58155b
--- /dev/null
+++ b/core/src/test/resources/ProcfsMetrics/487713/stat
@@ -0,0 +1 @@
+487713 ((Executor) task l)) D 474416 474398 474398 0 -1 4194368 5 0 0 0 0
0 0 0 25 5 1 0 1542745216 7469137920 120815 18446744073709551615
104424108929024 104424108932808 140734257079632 0 0 0 4 3 553671884 1 0 0 17 58
0 0 0 0 0 104424108940536 104424108941336 104424532111360 140734257083781
140734257085131 140734257085131 140734257102797 0
\ No newline at end of file
diff --git
a/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
b/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
index 77d782461a2e..caaf03d5d54e 100644
---
a/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
+++
b/core/src/test/scala/org/apache/spark/executor/ProcfsMetricsGetterSuite.scala
@@ -93,6 +93,18 @@ class ProcfsMetricsGetterSuite extends SparkFunSuite {
SparkEnv.set(originalSparkEnv)
}
}
+
+ test("SPARK-52776: Whitespace and parentheses in the comm field") {
+ val p = new ProcfsMetricsGetter(getTestResourcePath("ProcfsMetrics"))
+ var r = ProcfsMetrics(0, 0, 0, 0, 0, 0)
+ r = p.addProcfsMetricsFromOneProcess(r, 487713)
+ assert(r.jvmVmemTotal == 0)
+ assert(r.jvmRSSTotal == 0)
+ assert(r.pythonVmemTotal == 0)
+ assert(r.pythonRSSTotal == 0)
+ assert(r.otherVmemTotal == 7469137920L)
+ assert(r.otherRSSTotal == 494858240)
+ }
}
object ProcfsMetricsGetterSuite {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
