[
https://issues.apache.org/jira/browse/KAFKA-1710?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14173330#comment-14173330
]
Bhavesh Mistry commented on KAFKA-1710:
---------------------------------------
Here is out put of Yourkit:
{code}
Frozen threads found (potential deadlock)
It seems that the following threads have not changed their stack for more than
10 seconds.
These threads are possibly (but not necessarily!) in a deadlock or hung.
kafka-producer-network-thread <--- Frozen for at least 14 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.ready(Cluster,
long) RecordAccumulator.java:214
org.apache.kafka.clients.producer.internals.Sender.run(long) Sender.java:147
org.apache.kafka.clients.producer.internals.Sender.run() Sender.java:115
java.lang.Thread.run() Thread.java:744
pool-1-thread-106 <--- Frozen for at least 20 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-15 <--- Frozen for at least 13 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-161 <--- Frozen for at least 13 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-165 <--- Frozen for at least 17 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-172 <--- Frozen for at least 20 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-184 <--- Frozen for at least 11 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-26 <--- Frozen for at least 11 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-35 <--- Frozen for at least 14 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-37 <--- Frozen for at least 12 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-4 <--- Frozen for at least 12 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-42 <--- Frozen for at least 11 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-52 <--- Frozen for at least 11 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-55 <--- Frozen for at least 17 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-6 <--- Frozen for at least 11 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-82 <--- Frozen for at least 13 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-84 <--- Frozen for at least 12 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-88 <--- Frozen for at least 10 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
pool-1-thread-93 <--- Frozen for at least 14 sec
org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord, Callback)
KafkaProducer.java:238
org.kafka.test.TestNetworkDownProducer$MyProducer.run()
TestNetworkDownProducer.java:85
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
ThreadPoolExecutor.java:1145
java.util.concurrent.ThreadPoolExecutor$Worker.run() ThreadPoolExecutor.java:615
java.lang.Thread.run() Thread.java:744
{code}
> [New Java Producer Potential Deadlock] Producer Deadlock when all messages is
> being sent to single partition
> ------------------------------------------------------------------------------------------------------------
>
> Key: KAFKA-1710
> URL: https://issues.apache.org/jira/browse/KAFKA-1710
> Project: Kafka
> Issue Type: Bug
> Components: producer
> Environment: Development
> Reporter: Bhavesh Mistry
> Assignee: Jun Rao
> Priority: Critical
> Labels: performance
> Attachments: Screen Shot 2014-10-13 at 10.19.04 AM.png, Screen Shot
> 2014-10-15 at 9.09.06 PM.png, TestNetworkDownProducer.java
>
>
> Hi Kafka Dev Team,
> When I run the test to send message to single partition for 3 minutes or so
> on, I have encounter deadlock (please see the screen attached) and thread
> contention from YourKit profiling.
> Use Case:
> 1) Aggregating messages into same partition for metric counting.
> 2) Replicate Old Producer behavior for sticking to partition for 3 minutes.
> Here is output:
> Frozen threads found (potential deadlock)
>
> It seems that the following threads have not changed their stack for more
> than 10 seconds.
> These threads are possibly (but not necessarily!) in a deadlock or hung.
>
> pool-1-thread-128 <--- Frozen for at least 2m
> org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
> byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
> org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord,
> Callback) KafkaProducer.java:237
> org.kafka.test.TestNetworkDownProducer$MyProducer.run()
> TestNetworkDownProducer.java:84
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
> ThreadPoolExecutor.java:1145
> java.util.concurrent.ThreadPoolExecutor$Worker.run()
> ThreadPoolExecutor.java:615
> java.lang.Thread.run() Thread.java:744
> pool-1-thread-159 <--- Frozen for at least 2m 1 sec
> org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
> byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
> org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord,
> Callback) KafkaProducer.java:237
> org.kafka.test.TestNetworkDownProducer$MyProducer.run()
> TestNetworkDownProducer.java:84
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
> ThreadPoolExecutor.java:1145
> java.util.concurrent.ThreadPoolExecutor$Worker.run()
> ThreadPoolExecutor.java:615
> java.lang.Thread.run() Thread.java:744
> pool-1-thread-55 <--- Frozen for at least 2m
> org.apache.kafka.clients.producer.internals.RecordAccumulator.append(TopicPartition,
> byte[], byte[], CompressionType, Callback) RecordAccumulator.java:139
> org.apache.kafka.clients.producer.KafkaProducer.send(ProducerRecord,
> Callback) KafkaProducer.java:237
> org.kafka.test.TestNetworkDownProducer$MyProducer.run()
> TestNetworkDownProducer.java:84
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor$Worker)
> ThreadPoolExecutor.java:1145
> java.util.concurrent.ThreadPoolExecutor$Worker.run()
> ThreadPoolExecutor.java:615
> java.lang.Thread.run() Thread.java:744
> Thanks,
> Bhavesh
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
