[
https://issues.apache.org/jira/browse/STORM-1109?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Robert Joseph Evans updated STORM-1109:
---------------------------------------
Summary: Worker exists frequently due to java.net.SocketTimeoutException
(was: Supervisor exists frequently due to java.net.SocketTimeoutException)
> Worker exists frequently due to java.net.SocketTimeoutException
> ---------------------------------------------------------------
>
> Key: STORM-1109
> URL: https://issues.apache.org/jira/browse/STORM-1109
> Project: Apache Storm
> Issue Type: Bug
> Components: storm-kafka
> Affects Versions: 0.9.5
> Environment: Prod
> Reporter: rakesh
> Priority: Critical
>
> One of the supervisor is exiting frequently , following is the log
> 015-10-13T20:57:20.245+0530 k.c.SimpleConsumer [INFO] Reconnect due to socket
> error: null
> 2015-10-13T20:57:30.284+0530 b.s.util [ERROR] Async loop died!
> java.lang.RuntimeException: java.net.SocketTimeoutException
> at storm.kafka.KafkaUtils.fetchMessages(KafkaUtils.java:146)
> ~[stormjar.jar:na]
> at storm.kafka.PartitionManager.fill(PartitionManager.java:134)
> ~[stormjar.jar:na]
> at storm.kafka.PartitionManager.next(PartitionManager.java:108)
> ~[stormjar.jar:na]
> at storm.kafka.KafkaSpout.nextTuple(KafkaSpout.java:123)
> ~[stormjar.jar:na]
> at
> backtype.storm.daemon.executor$fn__6579$fn__6594$fn__6623.invoke(executor.clj:565)
> ~[storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.util$async_loop$fn__459.invoke(util.clj:463)
> ~[storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
> Caused by: java.net.SocketTimeoutException: null
> at
> sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:229)
> ~[na:1.7.0_67]
> at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
> ~[na:1.7.0_67]
> at
> java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385)
> ~[na:1.7.0_67]
> at kafka.utils.Utils$.read(Utils.scala:395) ~[stormjar.jar:na]
> at
> kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54)
> ~[stormjar.jar:na]
> at kafka.network.Receive$class.readCompletely(Transmission.scala:56)
> ~[stormjar.jar:na]
> at
> kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29)
> ~[stormjar.jar:na]
> at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:81)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:110)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:110)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:110)
> ~[stormjar.jar:na]
> at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:109)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:109)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:109)
> ~[stormjar.jar:na]
> at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33)
> ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:108)
> ~[stormjar.jar:na]
> at
> kafka.javaapi.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:48)
> ~[stormjar.jar:na]
> at storm.kafka.KafkaUtils.fetchMessages(KafkaUtils.java:141)
> ~[stormjar.jar:na]
> ... 7 common frames omitted
> 2015-10-13T20:57:30.285+0530 b.s.d.executor [ERROR]
> java.lang.RuntimeException: java.net.SocketTimeoutException
> at storm.kafka.KafkaUtils.fetchMessages(KafkaUtils.java:146)
> ~[stormjar.jar:na]
> at storm.kafka.PartitionManager.fill(PartitionManager.java:134)
> ~[stormjar.jar:na]
> at storm.kafka.PartitionManager.next(PartitionManager.java:108)
> ~[stormjar.jar:na]
> at storm.kafka.KafkaSpout.nextTuple(KafkaSpout.java:123)
> ~[stormjar.jar:na]
> at
> backtype.storm.daemon.executor$fn__6579$fn__6594$fn__6623.invoke(executor.clj:565)
> ~[storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.util$async_loop$fn__459.invoke(util.clj:463)
> ~[storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
> Caused by: java.net.SocketTimeoutException: null
> at
> sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:229)
> ~[na:1.7.0_67]
> at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
> ~[na:1.7.0_67]
> at
> java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385)
> ~[na:1.7.0_67]
> at kafka.utils.Utils$.read(Utils.scala:395) ~[stormjar.jar:na]
> at
> kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54)
> ~[stormjar.jar:na]
> at kafka.network.Receive$class.readCompletely(Transmission.scala:56)
> ~[stormjar.jar:na]
> at
> kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29)
> ~[stormjar.jar:na]
> at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:81)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:110)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:110)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:110)
> ~[stormjar.jar:na]
> at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:109)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:109)
> ~[stormjar.jar:na]
> at
> kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:109)
> ~[stormjar.jar:na]
> at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33)
> ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:108)
> ~[stormjar.jar:na]
> at
> kafka.javaapi.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:48)
> ~[stormjar.jar:na]
> at storm.kafka.KafkaUtils.fetchMessages(KafkaUtils.java:141)
> ~[stormjar.jar:na]
> ... 7 common frames omitted
> 2015-10-13T20:57:47.906+0530 b.s.util [ERROR] Halting process: ("Worker died")
> java.lang.RuntimeException: ("Worker died")
> at backtype.storm.util$exit_process_BANG_.doInvoke(util.clj:325)
> [storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.RestFn.invoke(RestFn.java:423) [clojure-1.5.1.jar:na]
> at
> backtype.storm.daemon.worker$fn__7028$fn__7029.invoke(worker.clj:497)
> [storm-core-0.9.5.jar:0.9.5]
> at
> backtype.storm.daemon.executor$mk_executor_data$fn__6480$fn__6481.invoke(executor.clj:240)
> [storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.util$async_loop$fn__459.invoke(util.clj:473)
> [storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
> 2015-10-13T20:57:47.907+0530 b.s.d.worker [INFO] Shutting down worker
> this is usually accompanied by the following error is supervisor.
> 2015-10-13T20:58:21.858+0530 b.s.d.supervisor [INFO] Shutting down
> fa862fbe-bfd2-4b53-9abb-cd951303ecb8:42ccaf83-6a17-4dcb-92e6-47416486806d
> 2015-10-13T20:58:21.869+0530 b.s.event [ERROR] Error when processing event
> java.io.IOException: . doesn't exist.
> at
> org.apache.commons.exec.DefaultExecutor.execute(DefaultExecutor.java:157)
> ~[commons-exec-1.1.jar:1.1]
> at
> org.apache.commons.exec.DefaultExecutor.execute(DefaultExecutor.java:147)
> ~[commons-exec-1.1.jar:1.1]
> at backtype.storm.util$exec_command_BANG_.invoke(util.clj:386)
> ~[storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.util$send_signal_to_process.invoke(util.clj:415)
> ~[storm-core-0.9.5.jar:0.9.5]
> at
> backtype.storm.util$kill_process_with_sig_term.invoke(util.clj:426)
> ~[storm-core-0.9.5.jar:0.9.5]
> at
> backtype.storm.daemon.supervisor$shutdown_worker.invoke(supervisor.clj:197)
> ~[storm-core-0.9.5.jar:0.9.5]
> at
> backtype.storm.daemon.supervisor$sync_processes.invoke(supervisor.clj:267)
> ~[storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.applyToHelper(AFn.java:161) [clojure-1.5.1.jar:na]
> at clojure.lang.AFn.applyTo(AFn.java:151) [clojure-1.5.1.jar:na]
> at clojure.core$apply.invoke(core.clj:619) ~[clojure-1.5.1.jar:na]
> at clojure.core$partial$fn__4190.doInvoke(core.clj:2396)
> ~[clojure-1.5.1.jar:na]
> at clojure.lang.RestFn.invoke(RestFn.java:397) ~[clojure-1.5.1.jar:na]
> at backtype.storm.event$event_manager$fn__2625.invoke(event.clj:40)
> ~[storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
> 2015-10-13T20:58:21.877+0530 b.s.util [ERROR] Halting process: ("Error when
> processing an event")
> java.lang.RuntimeException: ("Error when processing an event")
> at backtype.storm.util$exit_process_BANG_.doInvoke(util.clj:325)
> [storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.RestFn.invoke(RestFn.java:423) [clojure-1.5.1.jar:na]
> at backtype.storm.event$event_manager$fn__2625.invoke(event.clj:48)
> [storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
