Github user mridulm commented on a diff in the pull request:
https://github.com/apache/spark/pull/21895#discussion_r207493280
--- Diff:
core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala ---
@@ -161,6 +162,29 @@ private[history] class FsHistoryProvider(conf:
SparkConf, clock: Clock)
new HistoryServerDiskManager(conf, path, listing, clock)
}
+ private val blacklist = new ConcurrentHashMap[String, Long]
+
+ // Visible for testing
+ private[history] def isBlacklisted(path: Path): Boolean = {
+ blacklist.containsKey(path.getName)
+ }
+
+ private def blacklist(path: Path): Unit = {
+ blacklist.put(path.getName, clock.getTimeMillis())
+ }
+
+ /**
+ * Removes expired entries in the blacklist, according to the provided
`expireTimeInSeconds`.
+ */
+ private def clearBlacklist(expireTimeInSeconds: Long): Unit = {
+ val expiredThreshold = clock.getTimeMillis() - expireTimeInSeconds *
1000
+ val expired = new mutable.ArrayBuffer[String]
+ blacklist.asScala.foreach {
+ case (path, creationTime) if creationTime < expiredThreshold =>
expired += path
+ }
+ expired.foreach(blacklist.remove(_))
--- End diff --
Instead of this, why not simply:
```
blacklist.asScala.retain((_, creationTime) => creationTime >=
expiredThreshold)
```
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
