markusthoemmes commented on a change in pull request #3072: Enhance kafka
message provider
URL:
https://github.com/apache/incubator-openwhisk/pull/3072#discussion_r167402522
##########
File path:
common/scala/src/main/scala/whisk/connector/kafka/KafkaConsumerConnector.scala
##########
@@ -21,46 +21,91 @@ import java.util.Properties
import scala.collection.JavaConversions.iterableAsScalaIterable
import scala.collection.JavaConversions.seqAsJavaList
-import scala.concurrent.duration.Duration
-import scala.concurrent.duration.DurationInt
-import scala.concurrent.duration.FiniteDuration
-
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.kafka.common.serialization.ByteArrayDeserializer
-
+import org.apache.kafka.common.errors.{RetriableException, WakeupException}
+import akka.actor.ActorSystem
import whisk.common.Logging
import whisk.core.connector.MessageConsumer
-class KafkaConsumerConnector(kafkahost: String,
- groupid: String,
- topic: String,
- override val maxPeek: Int = Int.MaxValue,
- readeos: Boolean = true,
- sessionTimeout: FiniteDuration = 30.seconds,
- autoCommitInterval: FiniteDuration = 10.seconds,
- maxPollInterval: FiniteDuration =
5.minutes)(implicit logging: Logging)
+class KafkaConsumerConnector(
+ kafkahost: String,
+ groupid: String,
+ topic: String,
+ override val maxPeek: Int = Int.MaxValue,
+ readeos: Boolean = true,
+ sessionTimeout: FiniteDuration = 30.seconds,
+ autoCommitInterval: FiniteDuration = 10.seconds,
+ maxPollInterval: FiniteDuration = 5.minutes)(implicit logging: Logging,
actorSystem: ActorSystem)
extends MessageConsumer {
+ implicit val ec: ExecutionContext = actorSystem.dispatcher
+ private val gracefulWaitTime = 100.milliseconds.toMillis
+
/**
* Long poll for messages. Method returns once message are available but no
later than given
* duration.
*
* @param duration the maximum duration for the long poll
*/
- override def peek(duration: Duration = 500.milliseconds) = {
- val records = consumer.poll(duration.toMillis)
- records map { r =>
- (r.topic, r.partition, r.offset, r.value)
- }
+ override def peek(duration: Duration = 500.milliseconds,
+ retry: Int = 3): Iterable[(String, Int, Long,
Array[Byte])] = {
+ // Since kafka client can be infinitely blocked in poll(timeout), we
should interrupt it using the wakeup() method
Review comment:
Can you elaborate on why the poll could be blocked infinitely? In theory,
the timeout should limit the time it can be blocked? Is this for edge cases
(i.e. lost connection)?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
