I frequently get time-outs using heavy UI stuff, like dashboards with
mulitple widgets and just streams. My servers are heavily loaded just doing
inserts, so this is probably just graylog2-server just responding slowly,
but then the web-ui resets to search - thats not a great way to handle this.
I would love longer time-outs in the web-client - or maybe I can create a
new graylog2-server instance to servce the web-ui only?
I might create more elasticsearch nodes, as they seem to be using all 6
vcpus with 5000-10000 udp syslog msgs/s.
This is the error in web.log :
[ESC[31merrorESC[0m] lib.ApiClient - REST call failed
java.util.concurrent.TimeoutException: No response received after 5000
at
com.ning.http.client.providers.netty.NettyResponseFuture.get(NettyResponseFuture.java:245)
~[com.ning.async-http-client-1.7.18.jar:na]
at
lib.ApiClientImpl$ApiRequestBuilder.execute(ApiClientImpl.java:366)
~[graylog2-web-interface.graylog2-web-interface-0.20.1.jar:0.20.1]
at models.SavedSearchService.all(SavedSearchService.java:57)
~[graylog2-web-interface.graylog2-web-interface-0.20.1.jar:0.20.1]
at
controllers.api.SavedSearchesApiController.list(SavedSearchesApiController.java:52)
~[graylog2-web-interface.graylog2-web-interface-0.20.1.jar:0.20.1]
at
Routes$$anonfun$routes$1$$anonfun$applyOrElse$66$$anonfun$apply$196.apply(routes_routing.scala:1129)
~[graylog2-web-interface.graylog2-web-interface-0.20.1.jar:na]
at
Routes$$anonfun$routes$1$$anonfun$applyOrElse$66$$anonfun$apply$196.apply(routes_routing.scala:1129)
~[graylog2-web-interface.graylog2-web-interface-0.20.1.jar:na]
This looks to be related rrrors in server.log:
12:30:21,316 ERROR [NettyContainer] Uncaught exception during jersey
resource handling
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:51)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:487)
at
org.jboss.netty.channel.socket.nio.SocketSendBufferPool$UnpooledSendBuffer.transferTo(SocketSendBufferPool.java:203)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.write0(AbstractNioWorker.java:201)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.writeFromUserCode(AbstractNioWorker.java:146)
at
org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.handleAcceptedSocket(NioServerSocketPipelineSink.java:99)
at
org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink.eventSunk(NioServerSocketPipelineSink.java:36)
at org.jboss.netty.channel.Channels.write(Channels.java:725)
at
org.jboss.netty.handler.codec.oneone.OneToOneEncoder.doEncode(OneToOneEncoder.java:71)
at
org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:59)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:280)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleDownstream(ChunkedWriteHandler.java:121)
at org.jboss.netty.channel.Channels.write(Channels.java:704)
at org.jboss.netty.channel.Channels.write(Channels.java:671)
at org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:248)
at
org.graylog2.jersey.container.netty.NettyContainer$NettyResponseWriter.commit(NettyContainer.java:156)
at
org.glassfish.jersey.server.ContainerResponse.close(ContainerResponse.java:412)
at
org.glassfish.jersey.server.ServerRuntime$Responder.writeResponse(ServerRuntime.java:645)
at
org.glassfish.jersey.server.ServerRuntime$Responder.processResponse(ServerRuntime.java:381)
at
org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:371)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:262)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at
org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:318)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:236)
at
org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1010)
at
org.graylog2.jersey.container.netty.NettyContainer.messageReceived(NettyContainer.java:275)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleUpstream(ChunkedWriteHandler.java:142)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296)
at
org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:459)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:536)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
12:30:21,316 ERROR [NettyContainer] Uncaught exception during jersey
resource handling
java.nio.channels.ClosedChannelException
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.discard(ChunkedWriteHandler.java:168)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:285)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleDownstream(ChunkedWriteHandler.java:121)
at org.jboss.netty.channel.Channels.write(Channels.java:704)
at org.jboss.netty.channel.Channels.write(Channels.java:671)
at org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:248)
at
org.graylog2.jersey.container.netty.NettyContainer$NettyResponseWriter.commit(NettyContainer.java:156)
at
org.glassfish.jersey.server.ContainerResponse.close(ContainerResponse.java:412)
at
org.glassfish.jersey.server.ServerRuntime$Responder.writeResponse(ServerRuntime.java:645)
at
org.glassfish.jersey.server.ServerRuntime$Responder.processResponse(ServerRuntime.java:381)
at
org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:371)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:262)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at
org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:318)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:236)
at
org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1010)
at
org.graylog2.jersey.container.netty.NettyContainer.messageReceived(NettyContainer.java:275)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleUpstream(ChunkedWriteHandler.java:142)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296)
at
org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:459)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:536)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
12:30:21,317 INFO [NettyContainer] Not writing any response, channel is
already closed.
java.nio.channels.ClosedChannelException
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.discard(ChunkedWriteHandler.java:168)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:285)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleDownstream(ChunkedWriteHandler.java:121)
at org.jboss.netty.channel.Channels.write(Channels.java:704)
at org.jboss.netty.channel.Channels.write(Channels.java:671)
at org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:248)
at
org.graylog2.jersey.container.netty.NettyContainer$NettyResponseWriter.commit(NettyContainer.java:156)
at
org.glassfish.jersey.server.ContainerResponse.close(ContainerResponse.java:412)
at
org.glassfish.jersey.server.ServerRuntime$Responder.writeResponse(ServerRuntime.java:645)
at
org.glassfish.jersey.server.ServerRuntime$Responder.processResponse(ServerRuntime.java:381)
at
org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:371)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:262)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at
org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:318)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:236)
at
org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1010)
at
org.graylog2.jersey.container.netty.NettyContainer.messageReceived(NettyContainer.java:275)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleUpstream(ChunkedWriteHandler.java:142)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296)
at
org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:459)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:536)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
12:30:21,317 ERROR [NettyContainer] Uncaught exception during jersey
resource handling
java.nio.channels.ClosedChannelException
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.discard(ChunkedWriteHandler.java:168)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:192)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleDownstream(ChunkedWriteHandler.java:121)
at org.jboss.netty.channel.Channels.write(Channels.java:704)
at org.jboss.netty.channel.Channels.write(Channels.java:671)
at org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:248)
at
org.graylog2.jersey.container.netty.NettyContainer$NettyResponseWriter.commit(NettyContainer.java:157)
at
org.glassfish.jersey.server.ContainerResponse.close(ContainerResponse.java:412)
at
org.glassfish.jersey.server.ServerRuntime$Responder.writeResponse(ServerRuntime.java:645)
at
org.glassfish.jersey.server.ServerRuntime$Responder.processResponse(ServerRuntime.java:381)
at
org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:371)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:262)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at
org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:318)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:236)
at
org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1010)
at
org.graylog2.jersey.container.netty.NettyContainer.messageReceived(NettyContainer.java:275)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleUpstream(ChunkedWriteHandler.java:142)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296)
at
org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:459)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:536)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
12:30:21,317 INFO [NettyContainer] Not writing any response, channel is
already closed.
java.nio.channels.ClosedChannelException
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.discard(ChunkedWriteHandler.java:168)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.flush(ChunkedWriteHandler.java:192)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleDownstream(ChunkedWriteHandler.java:121)
at org.jboss.netty.channel.Channels.write(Channels.java:704)
at org.jboss.netty.channel.Channels.write(Channels.java:671)
at org.jboss.netty.channel.AbstractChannel.write(AbstractChannel.java:248)
at
org.graylog2.jersey.container.netty.NettyContainer$NettyResponseWriter.commit(NettyContainer.java:157)
at
org.glassfish.jersey.server.ContainerResponse.close(ContainerResponse.java:412)
at
org.glassfish.jersey.server.ServerRuntime$Responder.writeResponse(ServerRuntime.java:645)
at
org.glassfish.jersey.server.ServerRuntime$Responder.processResponse(ServerRuntime.java:381)
at
org.glassfish.jersey.server.ServerRuntime$Responder.process(ServerRuntime.java:371)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:262)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at
org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:318)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:236)
at
org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1010)
at
org.graylog2.jersey.container.netty.NettyContainer.messageReceived(NettyContainer.java:275)
at
org.jboss.netty.handler.stream.ChunkedWriteHandler.handleUpstream(ChunkedWriteHandler.java:142)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:296)
at
org.jboss.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:459)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.callDecode(ReplayingDecoder.java:536)
at
org.jboss.netty.handler.codec.replay.ReplayingDecoder.messageReceived(ReplayingDecoder.java:435)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108)
at
org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:318)
at
org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
--
You received this message because you are subscribed to the Google Groups
"graylog2" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
