Byoung joo, Lee created KAFKA-9357:
--------------------------------------
Summary: Error sending fetch request
Key: KAFKA-9357
URL: https://issues.apache.org/jira/browse/KAFKA-9357
Project: Kafka
Issue Type: Bug
Components: core, network
Affects Versions: 2.1.1
Environment: centos 7.4, 128gb memory, ssd 4tb disk, intel core 32cpu
Reporter: Byoung joo, Lee
It was 'OK' at first, but restarting after the first occurrence also reduced
the time it resolved. 6 months-> 1 month-> 1 day-> 2 days. The same thing
happens.
What's the solution?
I read some articles and set the replica.socket.receive.buffer.bytes = 33554432
value, but to no avail.
[2019-12-28 23:57:05,153] WARN [ReplicaFetcher replicaId=3, leaderId=2,
fetcherId=0] Error in response for fetch request (type=FetchRequest,
replicaId=3, maxWait=500, minBytes=1, maxBytes=10485760,
fetchData=\{logging.client.distribution-2=(offset=8880156,
logStartOffset=7704566, maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.protocolerr-3=(offset=446249750, logStartOffset=415015689,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.incoming-2=(offset=71425, logStartOffset=71425,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.err.protocolretry-0=(offset=8732169, logStartOffset=8732169,
maxBytes=1048576, currentLeaderEpoch=Optional[7]),
logging.client.protocolerr-0=(offset=446281548, logStartOffset=415013596,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.reception-4=(offset=607737872, logStartOffset=548501721,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.reception-1=(offset=607755530, logStartOffset=548502187,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.outgoing-5=(offset=9667, logStartOffset=9667, maxBytes=1048576,
currentLeaderEpoch=Optional[9]), queuing.device.button-0=(offset=2211009211,
logStartOffset=1982602128, maxBytes=1048576, currentLeaderEpoch=Optional[13]),
queuing.app.biz-0=(offset=0, logStartOffset=0, maxBytes=1048576,
currentLeaderEpoch=Optional[7]), logging.client.authfailure-5=(offset=1716617,
logStartOffset=1461528, maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.authfailure-2=(offset=1718246, logStartOffset=1463392,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.outgoing-2=(offset=9667, logStartOffset=9667, maxBytes=1048576,
currentLeaderEpoch=Optional[9]), queuing.device.biz-0=(offset=142402382,
logStartOffset=130377493, maxBytes=1048576, currentLeaderEpoch=Optional[13]),
logging.client.distribution-5=(offset=8883392, logStartOffset=7706253,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.incoming-5=(offset=71422, logStartOffset=71422,
maxBytes=1048576, currentLeaderEpoch=Optional[9])},
isolationLevel=READ_UNCOMMITTED, toForget=, metadata=(sessionId=507530224,
epoch=INITIAL)) (kafka.server.ReplicaFetcherThread)[2019-12-28 23:57:05,153]
WARN [ReplicaFetcher replicaId=3, leaderId=2, fetcherId=0] Error in response
for fetch request (type=FetchRequest, replicaId=3, maxWait=500, minBytes=1,
maxBytes=10485760, fetchData=\{logging.client.distribution-2=(offset=8880156,
logStartOffset=7704566, maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.protocolerr-3=(offset=446249750, logStartOffset=415015689,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.incoming-2=(offset=71425, logStartOffset=71425,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.err.protocolretry-0=(offset=8732169, logStartOffset=8732169,
maxBytes=1048576, currentLeaderEpoch=Optional[7]),
logging.client.protocolerr-0=(offset=446281548, logStartOffset=415013596,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.reception-4=(offset=607737872, logStartOffset=548501721,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.reception-1=(offset=607755530, logStartOffset=548502187,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.outgoing-5=(offset=9667, logStartOffset=9667, maxBytes=1048576,
currentLeaderEpoch=Optional[9]), queuing.device.button-0=(offset=2211009211,
logStartOffset=1982602128, maxBytes=1048576, currentLeaderEpoch=Optional[13]),
queuing.app.biz-0=(offset=0, logStartOffset=0, maxBytes=1048576,
currentLeaderEpoch=Optional[7]), logging.client.authfailure-5=(offset=1716617,
logStartOffset=1461528, maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.authfailure-2=(offset=1718246, logStartOffset=1463392,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.outgoing-2=(offset=9667, logStartOffset=9667, maxBytes=1048576,
currentLeaderEpoch=Optional[9]), queuing.device.biz-0=(offset=142402382,
logStartOffset=130377493, maxBytes=1048576, currentLeaderEpoch=Optional[13]),
logging.client.distribution-5=(offset=8883392, logStartOffset=7706253,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.incoming-5=(offset=71422, logStartOffset=71422,
maxBytes=1048576, currentLeaderEpoch=Optional[9])},
isolationLevel=READ_UNCOMMITTED, toForget=, metadata=(sessionId=507530224,
epoch=INITIAL)) (kafka.server.ReplicaFetcherThread)java.io.IOException:
Connection to 2 was disconnected before the response was read at
org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:97)
at
kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:97)
at
kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190)
at
kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:241)
at
kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:130)
at
kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:129)
at scala.Option.foreach(Option.scala:257) at
kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129)
at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111)
at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82)[2019-12-28
23:57:37,186] INFO [ReplicaFetcher replicaId=3, leaderId=2, fetcherId=0] Error
sending fetch request (sessionId=507530224, epoch=INITIAL) to node 2:
java.io.IOException: Connection to 2 was disconnected before the response was
read. (org.apache.kafka.clients.FetchSessionHandler)[2019-12-28 23:57:37,187]
WARN [ReplicaFetcher replicaId=3, leaderId=2, fetcherId=0] Error in response
for fetch request (type=FetchRequest, replicaId=3, maxWait=500, minBytes=1,
maxBytes=10485760, fetchData=\{logging.client.distribution-2=(offset=8880156,
logStartOffset=7704566, maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.protocolerr-3=(offset=446249750, logStartOffset=415015689,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.incoming-2=(offset=71425, logStartOffset=71425,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.err.protocolretry-0=(offset=8732169, logStartOffset=8732169,
maxBytes=1048576, currentLeaderEpoch=Optional[7]),
logging.client.protocolerr-0=(offset=446281548, logStartOffset=415013596,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.reception-4=(offset=607737872, logStartOffset=548501721,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.reception-1=(offset=607755530, logStartOffset=548502187,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.outgoing-5=(offset=9667, logStartOffset=9667, maxBytes=1048576,
currentLeaderEpoch=Optional[9]), queuing.device.button-0=(offset=2211009211,
logStartOffset=1982602128, maxBytes=1048576, currentLeaderEpoch=Optional[13]),
queuing.app.biz-0=(offset=0, logStartOffset=0, maxBytes=1048576,
currentLeaderEpoch=Optional[7]), logging.client.authfailure-5=(offset=1716617,
logStartOffset=1461528, maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.authfailure-2=(offset=1718246, logStartOffset=1463392,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.outgoing-2=(offset=9667, logStartOffset=9667, maxBytes=1048576,
currentLeaderEpoch=Optional[9]), queuing.device.biz-0=(offset=142402382,
logStartOffset=130377493, maxBytes=1048576, currentLeaderEpoch=Optional[13]),
logging.client.distribution-5=(offset=8883392, logStartOffset=7706253,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.incoming-5=(offset=71422, logStartOffset=71422,
maxBytes=1048576, currentLeaderEpoch=Optional[9])},
isolationLevel=READ_UNCOMMITTED, toForget=, metadata=(sessionId=507530224,
epoch=INITIAL)) (kafka.server.ReplicaFetcherThread)java.io.IOException:
Connection to 2 was disconnected before the response was read at
org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:97)
at
kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:97)
at
kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190)
at
kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:241)
at
kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:130)
at
kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:129)
at scala.Option.foreach(Option.scala:257) at
kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129)
at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111)
at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82)[2019-12-28
23:58:09,196] INFO [ReplicaFetcher replicaId=3, leaderId=2, fetcherId=0] Error
sending fetch request (sessionId=507530224, epoch=INITIAL) to node 2:
java.io.IOException: Connection to 2 was disconnected before the response was
read. (org.apache.kafka.clients.FetchSessionHandler)[2019-12-28 23:58:09,196]
WARN [ReplicaFetcher replicaId=3, leaderId=2, fetcherId=0] Error in response
for fetch request (type=FetchRequest, replicaId=3, maxWait=500, minBytes=1,
maxBytes=10485760, fetchData=\{logging.client.distribution-2=(offset=8880156,
logStartOffset=7704566, maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.protocolerr-3=(offset=446249750, logStartOffset=415015689,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.incoming-2=(offset=71425, logStartOffset=71425,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.err.protocolretry-0=(offset=8732169, logStartOffset=8732169,
maxBytes=1048576, currentLeaderEpoch=Optional[7]),
logging.client.protocolerr-0=(offset=446281548, logStartOffset=415013596,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.reception-4=(offset=607737872, logStartOffset=548501721,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.reception-1=(offset=607755530, logStartOffset=548502187,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.outgoing-5=(offset=9667, logStartOffset=9667, maxBytes=1048576,
currentLeaderEpoch=Optional[9]), queuing.device.button-0=(offset=2211009211,
logStartOffset=1982602128, maxBytes=1048576, currentLeaderEpoch=Optional[13]),
queuing.app.biz-0=(offset=0, logStartOffset=0, maxBytes=1048576,
currentLeaderEpoch=Optional[7]), logging.client.authfailure-5=(offset=1716617,
logStartOffset=1461528, maxBytes=1048576, currentLeaderEpoch=Optional[9]),
logging.client.authfailure-2=(offset=1718246, logStartOffset=1463392,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.outgoing-2=(offset=9667, logStartOffset=9667, maxBytes=1048576,
currentLeaderEpoch=Optional[9]), queuing.device.biz-0=(offset=142402382,
logStartOffset=130377493, maxBytes=1048576, currentLeaderEpoch=Optional[13]),
logging.client.distribution-5=(offset=8883392, logStartOffset=7706253,
maxBytes=1048576, currentLeaderEpoch=Optional[9]),
tracking.amount.incoming-5=(offset=71422, logStartOffset=71422,
maxBytes=1048576, currentLeaderEpoch=Optional[9])},
isolationLevel=READ_UNCOMMITTED, toForget=, metadata=(sessionId=507530224,
epoch=INITIAL)) (kafka.server.ReplicaFetcherThread)java.io.IOException:
Connection to 2 was disconnected before the response was read at
org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:97)
at
kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:97)
at
kafka.server.ReplicaFetcherThread.fetchFromLeader(ReplicaFetcherThread.scala:190)
at
kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:241)
at
kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:130)
at
kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:129)
at scala.Option.foreach(Option.scala:257) at
kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:129)
at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:111)
at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82)
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
