Jeffrey created USERGRID-1045:
----------------------------------
Summary: Ensure a queue capacity rejection is handled for ES
SEARCH requests
Key: USERGRID-1045
URL: https://issues.apache.org/jira/browse/USERGRID-1045
Project: Usergrid
Issue Type: Story
Reporter: Jeffrey
Currently we get this stack trace. Given how we were (mis)handling the bulk
queue overflow errors we should double-check how we handle the search request
case(s).
2015-10-06 22:03:15,116
[elasticsearch[default][clusterService#updateTask][T#1]] INFO
org.elasticsearch.cluster.service- [default] added
{[default][RonMuMoxRJ2e73K_zy_qZw][eut027wo][inet[/10.16.3.152:9300]]{client=true,
data=false},}, reason: zen-disco-receive(from master
[[ees000wo][55dT8lX5RxeSDk8UZy-2vA][ees000wo][inet[/10.16.3.164:9300]]{data=false,
master=true}])
2015-10-06 22:03:19,512 [QueueConsumer_2] ERROR
org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl- Unable to
communicate with Elasticsearch
org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected
execution (queue capacity 1) on
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$2@50664ee
at
org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:62)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.finishHim(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$100(TransportSearchQueryThenFetchAction.java:64)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.onResult(TransportSearchQueryThenFetchAction.java:115)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.onResult(TransportSearchQueryThenFetchAction.java:109)
at
org.elasticsearch.search.action.SearchServiceTransportAction$18.handleResponse(SearchServiceTransportAction.java:465)
at
org.elasticsearch.search.action.SearchServiceTransportAction$18.handleResponse(SearchServiceTransportAction.java:456)
at
org.elasticsearch.transport.netty.MessageChannelHandler.handleResponse(MessageChannelHandler.java:158)
at
org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:127)
at
org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at
org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
at
org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462)
at
org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443)
at
org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303)
at
org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at
org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at
org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at
org.elasticsearch.common.netty.channel.socket.oio.OioWorker.process(OioWorker.java:71)
at
org.elasticsearch.common.netty.channel.socket.oio.AbstractOioWorker.run(AbstractOioWorker.java:73)
at
org.elasticsearch.common.netty.channel.socket.oio.OioWorker.run(OioWorker.java:51)
at
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
2015-10-06 22:03:19,516 [QueueConsumer_2] ERROR
org.apache.usergrid.corepersistence.asyncevents.AmazonAsyncEventService- failed
to run index
org.elasticsearch.common.util.concurrent.EsRejectedExecutionException: rejected
execution (queue capacity 1) on
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$2@50664ee
at
org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:62)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.finishHim(TransportSearchQueryThenFetchAction.java:141)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$100(TransportSearchQueryThenFetchAction.java:64)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.onResult(TransportSearchQueryThenFetchAction.java:115)
at
org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.onResult(TransportSearchQueryThenFetchAction.java:109)
at
org.elasticsearch.search.action.SearchServiceTransportAction$18.handleResponse(SearchServiceTransportAction.java:465)
at
org.elasticsearch.search.action.SearchServiceTransportAction$18.handleResponse(SearchServiceTransportAction.java:456)
at
org.elasticsearch.transport.netty.MessageChannelHandler.handleResponse(MessageChannelHandler.java:158)
at
org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:127)
at
org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
at
org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
at
org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462)
at
org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443)
at
org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303)
at
org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
at
org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
at
org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
at
org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
at
org.elasticsearch.common.netty.channel.socket.oio.OioWorker.process(OioWorker.java:71)
at
org.elasticsearch.common.netty.channel.socket.oio.AbstractOioWorker.run(AbstractOioWorker.java:73)
at
org.elasticsearch.common.netty.channel.socket.oio.OioWorker.run(OioWorker.java:51)
at
org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
at
org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
