Github user koeninger commented on a diff in the pull request:
https://github.com/apache/spark/pull/18143#discussion_r119371920
--- Diff:
external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/CachedKafkaConsumer.scala
---
@@ -109,34 +113,38 @@ object CachedKafkaConsumer extends Logging {
private case class CacheKey(groupId: String, topic: String, partition:
Int)
- // Don't want to depend on guava, don't want a cleanup thread, use a
simple LinkedHashMap
- private var cache: ju.LinkedHashMap[CacheKey, CachedKafkaConsumer[_, _]]
= null
+ private var cache: Cache[CacheKey, CachedKafkaConsumer[_, _]] = null
/** Must be called before get, once per JVM, to configure the cache.
Further calls are ignored */
def init(
initialCapacity: Int,
maxCapacity: Int,
loadFactor: Float): Unit = CachedKafkaConsumer.synchronized {
- if (null == cache) {
- logInfo(s"Initializing cache $initialCapacity $maxCapacity
$loadFactor")
- cache = new ju.LinkedHashMap[CacheKey, CachedKafkaConsumer[_, _]](
- initialCapacity, loadFactor, true) {
- override def removeEldestEntry(
- entry: ju.Map.Entry[CacheKey, CachedKafkaConsumer[_, _]]):
Boolean = {
- if (this.size > maxCapacity) {
- try {
- entry.getValue.consumer.close()
- } catch {
- case x: KafkaException =>
- logError("Error closing oldest Kafka consumer", x)
- }
- true
- } else {
- false
- }
+ val duration =
SparkEnv.get.conf.getTimeAsMs("spark.sql.kafkaConsumerCache.timeout", "30m")
+ val removalListener = new RemovalListener[CacheKey,
CachedKafkaConsumer[_, _]] {
+ override def onRemoval(n: RemovalNotification[CacheKey,
CachedKafkaConsumer[_, _]]): Unit = {
+ n.getCause match {
+ case RemovalCause.SIZE =>
+ logWarning(
+ s"Evicting consumer ${n.getKey}, due to size limit reached.
Capacity: $maxCapacity.")
+ case _ => logDebug(s"Evicting consumer ${n.getKey}, cause:
${n.getCause}")
+ }
+ try {
+ n.getValue.close()
+ } catch {
+ case NonFatal(e) =>
+ logWarning(s"Error in closing Kafka consumer: ${n.getKey}," +
+ s" evicted from cache due to ${n.getCause}", e)
}
}
}
+ if (cache == null) {
--- End diff --
Shouldn't the whole function be guarded by this if statement? Is there any
reason to construct a removal listener otherwise?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
