Jeffrey created USERGRID-442:
---------------------------------
Summary: Consistent Pausing across all Tomcat nodes
Key: USERGRID-442
URL: https://issues.apache.org/jira/browse/USERGRID-442
Project: Usergrid
Issue Type: Story
Reporter: Jeffrey
Assignee: Shawn Feldman
Priority: Critical
I have seen as many as 16 tomcat nodes pausing at the same time across the
cluster. I took a stack trace and have attached it.
This one jumped out to me:
"http-bio-8080-exec-2" daemon prio=10 tid=0x00007f0cbc003800 nid=0x20ff waiting
on condition [0x00007f0d370dd000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007ee1de4d8> (a
org.elasticsearch.common.util.concurrent.BaseFuture$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303)
at
org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:274)
at
org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:113)
at
org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:45)
at
org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.search(EsEntityIndexImpl.java:357)
at
org.apache.usergrid.corepersistence.CpRelationManager.searchCollection(CpRelationManager.java:963)
at
org.apache.usergrid.corepersistence.CpEntityManager.searchCollection(CpEntityManager.java:624)
at
org.apache.usergrid.corepersistence.CpEntityManagerFactory.lookupApplication(CpEntityManagerFactory.java:433)
at
org.apache.usergrid.rest.organizations.OrganizationResource.getApplicationByName(OrganizationResource.java:137)
at sun.reflect.GeneratedMethodAccessor113.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at
com.sun.jersey.server.impl.uri.rules.SubLocatorRule.dispatch(SubLocatorRule.java:197)
at
com.sun.jersey.server.impl.uri.rules.SubLocatorRule.invokeSubLocator(SubLocatorRule.java:183)
at
com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:110)
at
com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
at
com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137)
at
com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
at
com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)
at
com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)
at
com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)
at
com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542)
at
com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473)
at
com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419)
at
com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409)
at
com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409)
at
com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:540)
at
com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:909)
at
com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:857)
at
com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:811)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at
org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449)
at
org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365)
at
org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90)
at
org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83)
at
org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383)
at
org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362)
at
org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
at
org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at
org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at
org.apache.usergrid.rest.filters.ContentTypeFilter.doFilter(ContentTypeFilter.java:92)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:503)
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170)
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at
org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421)
at
org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1070)
at
org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611)
at
org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:314)
- locked <0x00000007eda23278> (a
org.apache.tomcat.util.net.SocketWrapper)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:722)
and Todd called out this one:
"collectiontasks-18" daemon prio=10 tid=0x00007f0cfc004000 nid=0x21b7 waiting
on condition [0x00007f0cb87c6000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007ed32ccf0> (a
org.elasticsearch.common.util.concurrent.BaseFuture$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:994)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1303)
at
org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:274)
at org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:113)
at
org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:45)
at
org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl.deletePreviousVersions(EsEntityIndexImpl.java:533)
at
org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler.versionCreated(EntityVersionCreatedHandler.java:67)
at
org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask.fireEvents(EntityVersionCreatedTask.java:97)
at
org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask.call(EntityVersionCreatedTask.java:83)
at
org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask.call(EntityVersionCreatedTask.java:38)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
