[
https://issues.apache.org/jira/browse/USERGRID-1045?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Jeffrey updated USERGRID-1045:
-------------------------------
Sprint: Critical (was: Usergrid 32)
> Ensure a queue capacity rejection is handled for ES SEARCH requests
> -------------------------------------------------------------------
>
> Key: USERGRID-1045
> URL: https://issues.apache.org/jira/browse/USERGRID-1045
> Project: Usergrid
> Issue Type: Story
> Reporter: Jeffrey
> Priority: Critical
>
> Currently we get this stack trace. Given how we were (mis)handling the bulk
> queue overflow errors we should double-check how we handle the search request
> case(s).
> 2015-10-06 22:03:15,116
> [elasticsearch[default][clusterService#updateTask][T#1]] INFO
> org.elasticsearch.cluster.service- [default] added
> {[default][RonMuMoxRJ2e73K_zy_qZw][eut027wo][inet[/10.16.3.152:9300]]{client=true,
> data=false},}, reason: zen-disco-receive(from master
> [[ees000wo][55dT8lX5RxeSDk8UZy-2vA][ees000wo][inet[/10.16.3.164:9300]]{data=false,
> master=true}])
> 2015-10-06 22:03:19,512 [QueueConsumer_2] ERROR
> org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl- Unable to
> communicate with Elasticsearch
> org.elasticsearch.common.util.concurrent.EsRejectedExecutionException:
> rejected execution (queue capacity 1) on
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$2@50664ee
> at
> org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:62)
> at
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
> at
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
> at
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.finishHim(TransportSearchQueryThenFetchAction.java:141)
> at
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$100(TransportSearchQueryThenFetchAction.java:64)
> at
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.onResult(TransportSearchQueryThenFetchAction.java:115)
> at
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.onResult(TransportSearchQueryThenFetchAction.java:109)
> at
> org.elasticsearch.search.action.SearchServiceTransportAction$18.handleResponse(SearchServiceTransportAction.java:465)
> at
> org.elasticsearch.search.action.SearchServiceTransportAction$18.handleResponse(SearchServiceTransportAction.java:456)
> at
> org.elasticsearch.transport.netty.MessageChannelHandler.handleResponse(MessageChannelHandler.java:158)
> at
> org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:127)
> at
> org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> at
> org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> at
> org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
> at
> org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
> at
> org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462)
> at
> org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443)
> at
> org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303)
> at
> org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> at
> org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> at
> org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
> at
> org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
> at
> org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
> at
> org.elasticsearch.common.netty.channel.socket.oio.OioWorker.process(OioWorker.java:71)
> at
> org.elasticsearch.common.netty.channel.socket.oio.AbstractOioWorker.run(AbstractOioWorker.java:73)
> at
> org.elasticsearch.common.netty.channel.socket.oio.OioWorker.run(OioWorker.java:51)
> at
> org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> 2015-10-06 22:03:19,516 [QueueConsumer_2] ERROR
> org.apache.usergrid.corepersistence.asyncevents.AmazonAsyncEventService-
> failed to run index
> org.elasticsearch.common.util.concurrent.EsRejectedExecutionException:
> rejected execution (queue capacity 1) on
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$2@50664ee
> at
> org.elasticsearch.common.util.concurrent.EsAbortPolicy.rejectedExecution(EsAbortPolicy.java:62)
> at
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
> at
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
> at
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.finishHim(TransportSearchQueryThenFetchAction.java:141)
> at
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction.access$100(TransportSearchQueryThenFetchAction.java:64)
> at
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.onResult(TransportSearchQueryThenFetchAction.java:115)
> at
> org.elasticsearch.action.search.type.TransportSearchQueryThenFetchAction$AsyncAction$1.onResult(TransportSearchQueryThenFetchAction.java:109)
> at
> org.elasticsearch.search.action.SearchServiceTransportAction$18.handleResponse(SearchServiceTransportAction.java:465)
> at
> org.elasticsearch.search.action.SearchServiceTransportAction$18.handleResponse(SearchServiceTransportAction.java:456)
> at
> org.elasticsearch.transport.netty.MessageChannelHandler.handleResponse(MessageChannelHandler.java:158)
> at
> org.elasticsearch.transport.netty.MessageChannelHandler.messageReceived(MessageChannelHandler.java:127)
> at
> org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> at
> org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> at
> org.elasticsearch.common.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791)
> at
> org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:296)
> at
> org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.unfoldAndFireMessageReceived(FrameDecoder.java:462)
> at
> org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:443)
> at
> org.elasticsearch.common.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303)
> at
> org.elasticsearch.common.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70)
> at
> org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564)
> at
> org.elasticsearch.common.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559)
> at
> org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:268)
> at
> org.elasticsearch.common.netty.channel.Channels.fireMessageReceived(Channels.java:255)
> at
> org.elasticsearch.common.netty.channel.socket.oio.OioWorker.process(OioWorker.java:71)
> at
> org.elasticsearch.common.netty.channel.socket.oio.AbstractOioWorker.run(AbstractOioWorker.java:73)
> at
> org.elasticsearch.common.netty.channel.socket.oio.OioWorker.run(OioWorker.java:51)
> at
> org.elasticsearch.common.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
> at
> org.elasticsearch.common.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
