This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ff2a822fc4f [SPARK-39224][CORE] Lower general ProcfsMetricsGetter
error log levels except `/proc/` lookup error
ff2a822fc4f is described below
commit ff2a822fc4fe27f6d5e07aedb087678eeaf69cae
Author: Angerszhuuuu <angers....@gmail.com>
AuthorDate: Wed May 18 20:50:40 2022 -0700
[SPARK-39224][CORE] Lower general ProcfsMetricsGetter error log levels
except `/proc/` lookup error
### What changes were proposed in this pull request?
There are many noisy warning logs such as
```
22/05/18 16:48:58 WARN ProcfsMetricsGetter: There was a problem with
reading the stat file of the process.
java.io.FileNotFoundException: /proc/50371/stat (No such file or directory)
at java.io.FileInputStream.open0(Native Method)
at java.io.FileInputStream.open(FileInputStream.java:195)
at java.io.FileInputStream.<init>(FileInputStream.java:138)
at
org.apache.spark.executor.ProcfsMetricsGetter.openReader$1(ProcfsMetricsGetter.scala:174)
at
org.apache.spark.executor.ProcfsMetricsGetter.$anonfun$addProcfsMetricsFromOneProcess$1(ProcfsMetricsGetter.scala:176)
at org.apache.spark.util.Utils$.tryWithResource(Utils.scala:2647)
at
org.apache.spark.executor.ProcfsMetricsGetter.addProcfsMetricsFromOneProcess(ProcfsMetricsGetter.scala:176)
at
org.apache.spark.executor.ProcfsMetricsGetter.$anonfun$computeAllMetrics$1(ProcfsMetricsGetter.scala:216)
at
scala.runtime.java8.JFunction1$mcVI$sp.apply(JFunction1$mcVI$sp.java:23)
at scala.collection.immutable.Set$Set2.foreach(Set.scala:132)
at
org.apache.spark.executor.ProcfsMetricsGetter.computeAllMetrics(ProcfsMetricsGetter.scala:214)
at
org.apache.spark.metrics.ProcessTreeMetrics$.getMetricValues(ExecutorMetricType.scala:93)
at
org.apache.spark.executor.ExecutorMetrics$.$anonfun$getCurrentMetrics$1(ExecutorMetrics.scala:103)
at
org.apache.spark.executor.ExecutorMetrics$.$anonfun$getCurrentMetrics$1$adapted(ExecutorMetrics.scala:102)
at scala.collection.Iterator.foreach(Iterator.scala:941)
at scala.collection.Iterator.foreach$(Iterator.scala:941)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at
org.apache.spark.executor.ExecutorMetrics$.getCurrentMetrics(ExecutorMetrics.scala:102)
at
org.apache.spark.SparkContext.reportHeartBeat(SparkContext.scala:2578)
at
org.apache.spark.SparkContext.$anonfun$new$30(SparkContext.scala:587)
at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at
org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2022)
at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
```
Many user complain about this. We can ignore this by default.
### Why are the changes needed?
Ignore noisy logs
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
MT
Closes #36596 from AngersZhuuuu/IGNORE-ProcfsMetricsGetter-MSG.
Authored-by: Angerszhuuuu <angers....@gmail.com>
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
.../scala/org/apache/spark/executor/ProcfsMetricsGetter.scala | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
index 5682a21e956..547d9df4e4f 100644
--- a/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ProcfsMetricsGetter.scala
@@ -77,7 +77,7 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String =
"/proc/") extends L
}
catch {
case e: SparkException =>
- logWarning("Exception when trying to compute process tree." +
+ logDebug("Exception when trying to compute process tree." +
" As a result reporting of ProcessTree metrics is stopped", e)
isAvailable = false
-1
@@ -94,7 +94,7 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String =
"/proc/") extends L
Integer.parseInt(out.split("\n")(0))
} catch {
case e: Exception =>
- logWarning("Exception when trying to compute pagesize, as a" +
+ logDebug("Exception when trying to compute pagesize, as a" +
" result reporting of ProcessTree metrics is stopped")
isAvailable = false
0
@@ -153,7 +153,7 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String
= "/proc/") extends L
childPidsInInt
} catch {
case e: Exception =>
- logWarning("Exception when trying to compute process tree." +
+ logDebug("Exception when trying to compute process tree." +
" As a result reporting of ProcessTree metrics is stopped.", e)
isAvailable = false
mutable.ArrayBuffer.empty[Int]
@@ -199,7 +199,7 @@ private[spark] class ProcfsMetricsGetter(procfsDir: String
= "/proc/") extends L
}
} catch {
case f: IOException =>
- logWarning("There was a problem with reading" +
+ logDebug("There was a problem with reading" +
" the stat file of the process. ", f)
throw f
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
