This is an automated email from the ASF dual-hosted git repository. kabhwan pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
commit 15ed3121994d4c7e6f1db7e0b5b2cea5565b495a Author: Yan Xiaole <[email protected]> AuthorDate: Sun Aug 9 16:47:31 2020 -0700 [SPARK-32557][CORE] Logging and swallowing the exception per entry in History server ### What changes were proposed in this pull request? This PR adds a try catch wrapping the History server scan logic to log and swallow the exception per entry. ### Why are the changes needed? As discussed in #29350 , one entry failure shouldn't affect others. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manually tested. Closes #29374 from yanxiaole/SPARK-32557. Authored-by: Yan Xiaole <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]> --- .../scala/org/apache/spark/deploy/history/FsHistoryProvider.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index c262152..f5e7c4fa 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -27,6 +27,7 @@ import java.util.zip.ZipOutputStream import scala.collection.JavaConverters._ import scala.collection.mutable import scala.io.Source +import scala.util.control.NonFatal import scala.xml.Node import com.fasterxml.jackson.annotation.JsonIgnore @@ -528,7 +529,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) case _: FileNotFoundException => false } - case _: FileNotFoundException => + case NonFatal(e) => + logWarning(s"Error while filtering log ${reader.rootPath}", e) false } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
