[
https://issues.apache.org/jira/browse/USERGRID-442?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Jeffrey resolved USERGRID-442.
-------------------------------
Resolution: Fixed
The problem here was related to several factors in ES, one of which was the
NewRatio. We tuned the GC settings and we were able to bypass this.
> Investigate ElasticSearch timeout such that a long-running GC does not pause
> all ES and Tomcat
> ----------------------------------------------------------------------------------------------
>
> Key: USERGRID-442
> URL: https://issues.apache.org/jira/browse/USERGRID-442
> Project: Usergrid
> Issue Type: Bug
> Reporter: Jeffrey
> Priority: Minor
> Attachments: paused_threads.txt
>
>
> I have seen as many as 16 tomcat nodes pausing at the same time across the
> cluster. I took a stack trace and have attached it.
> This one jumped out to me:
> "http-bio-8080-exec-2" daemon prio=10 tid=0x00007f0cbc003800 nid=0x20ff
> waiting on condition [0x00007f0d370dd000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000007ee1de4d8> (a
> org.elasticsearch.common.util.concurrent.BaseFuture$Sync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303)
> at
> org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:274)
> at
> org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:113)
> at
> org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:45)
> at
> org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.search(EsEntityIndexImpl.java:357)
> at
> org.apache.usergrid.corepersistence.CpRelationManager.searchCollection(CpRelationManager.java:963)
> at
> org.apache.usergrid.corepersistence.CpEntityManager.searchCollection(CpEntityManager.java:624)
> at
> org.apache.usergrid.corepersistence.CpEntityManagerFactory.lookupApplication(CpEntityManagerFactory.java:433)
> at
> org.apache.usergrid.rest.organizations.OrganizationResource.getApplicationByName(OrganizationResource.java:137)
> at sun.reflect.GeneratedMethodAccessor113.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:601)
> at
> com.sun.jersey.server.impl.uri.rules.SubLocatorRule.dispatch(SubLocatorRule.java:197)
> at
> com.sun.jersey.server.impl.uri.rules.SubLocatorRule.invokeSubLocator(SubLocatorRule.java:183)
> at
> com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:110)
> at
> com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
> at
> com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137)
> at
> com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
> at
> com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)
> at
> com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
> at
> com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
> at
> com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542)
> at
> com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473)
> at
> com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419)
> at
> com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409)
> at
> com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409)
> at
> com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:540)
> at
> com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:909)
> at
> com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:857)
> at
> com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:811)
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
> at
> org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449)
> at
> org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365)
> at
> org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90)
> at
> org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83)
> at
> org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383)
> at
> org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362)
> at
> org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
> at
> org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
> at
> org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259)
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
> at
> org.apache.usergrid.rest.filters.ContentTypeFilter.doFilter(ContentTypeFilter.java:92)
> at
> org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
> at
> org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
> at
> org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
> at
> org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
> at
> org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:503)
> at
> org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170)
> at
> org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
> at
> org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
> at
> org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
> at
> org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421)
> at
> org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1070)
> at
> org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611)
> at
> org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:314)
> - locked <0x00000007eda23278> (a
> org.apache.tomcat.util.net.SocketWrapper)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at
> org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
> at java.lang.Thread.run(Thread.java:722)
> and Todd called out this one:
> "collectiontasks-18" daemon prio=10 tid=0x00007f0cfc004000 nid=0x21b7 waiting
> on condition [0x00007f0cb87c6000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000007ed32ccf0> (a
> org.elasticsearch.common.util.concurrent.BaseFuture$Sync)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303)
> at
> org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:274)
> at
> org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:113)
> at
> org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:45)
> at
> org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.deletePreviousVersions(EsEntityIndexImpl.java:533)
> at
> org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler.versionCreated(EntityVersionCreatedHandler.java:67)
> at
> org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask.fireEvents(EntityVersionCreatedTask.java:97)
> at
> org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask.call(EntityVersionCreatedTask.java:83)
> at
> org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask.call(EntityVersionCreatedTask.java:38)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:722)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
