[
https://issues.apache.org/jira/browse/OAK-7194?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16346322#comment-16346322
]
Kshitiz Garg commented on OAK-7194:
-----------------------------------
Thanks for your response [~mduerig]. We are going through the page you have
shared.
There is no deadlock per se, but we are seeing that threads are getting blocked
at this exact point as some of our runnable threads hold the required locks for
prolonged periods. One such stack trace of a runnable thread is:
{noformat}
10.157.213.11 [1517322418781] POST
/content/finance-leadership/us/en/member/blog/18/01/4-trends-will-impact-companies-in-2018-what-cfos-should-do/_jcr_content.activitydatacapture.activitydatacapture
HTTP/1.1
Stack Trace is:
java.lang.Thread.State: BLOCKED
at java.io.RandomAccessFile.readBytes(Native Method)
at java.io.RandomAccessFile.read(RandomAccessFile.java:377)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:436)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:416)
at
org.apache.jackrabbit.oak.segment.file.FileAccess$Random.read(FileAccess.java:113)
- locked <0x3cdc62dd> (a
org.apache.jackrabbit.oak.segment.file.FileAccess$Random)
at
org.apache.jackrabbit.oak.segment.file.TarReader.readEntry(TarReader.java:575)
at org.apache.jackrabbit.oak.segment.file.FileStore$8.call(FileStore.java:557)
at org.apache.jackrabbit.oak.segment.file.FileStore$8.call(FileStore.java:542)
at
org.apache.jackrabbit.oak.segment.SegmentCache.getSegment(SegmentCache.java:95)
at
org.apache.jackrabbit.oak.segment.file.FileStore.readSegment(FileStore.java:542)
at org.apache.jackrabbit.oak.segment.SegmentId.getSegment(SegmentId.java:125)
- locked <0x77f1643d> (a org.apache.jackrabbit.oak.segment.SegmentId)
at org.apache.jackrabbit.oak.segment.Record.getSegment(Record.java:70)
at org.apache.jackrabbit.oak.segment.MapRecord.getEntry(MapRecord.java:160)
at org.apache.jackrabbit.oak.segment.MapRecord.getEntry(MapRecord.java:192)
at org.apache.jackrabbit.oak.segment.MapRecord.getEntry(MapRecord.java:192)
at org.apache.jackrabbit.oak.segment.MapRecord.getEntry(MapRecord.java:192)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.getChildNode(SegmentNodeState.java:412)
at
org.apache.jackrabbit.oak.plugins.tree.impl.ImmutableTree.createChild(ImmutableTree.java:125)
at
org.apache.jackrabbit.oak.plugins.tree.impl.ImmutableTree.getChild(ImmutableTree.java:176)
at
org.apache.jackrabbit.oak.plugins.tree.impl.ImmutableTree.getChild(ImmutableTree.java:81)
at
org.apache.jackrabbit.oak.security.authorization.permission.PermissionUtil.getPrincipalRoot(PermissionUtil.java:83)
at
org.apache.jackrabbit.oak.security.authorization.permission.PermissionStoreImpl.getPrincipalRoot(PermissionStoreImpl.java:132)
at
org.apache.jackrabbit.oak.security.authorization.permission.PermissionStoreImpl.getNumEntries(PermissionStoreImpl.java:103)
at
org.apache.jackrabbit.oak.security.authorization.permission.PermissionEntryCache.getNumEntries(PermissionEntryCache.java:102)
at
org.apache.jackrabbit.oak.security.authorization.permission.PermissionEntryProviderImpl.init(PermissionEntryProviderImpl.java:79)
at
org.apache.jackrabbit.oak.security.authorization.permission.PermissionEntryProviderImpl.<init>(PermissionEntryProviderImpl.java:72)
at
org.apache.jackrabbit.oak.security.authorization.permission.CompiledPermissionImpl.<init>(CompiledPermissionImpl.java:112)
at
org.apache.jackrabbit.oak.security.authorization.permission.CompiledPermissionImpl.create(CompiledPermissionImpl.java:126)
at
org.apache.jackrabbit.oak.security.authorization.permission.PermissionProviderImpl.getCompiledPermissions(PermissionProviderImpl.java:162)
at
org.apache.jackrabbit.oak.security.authorization.permission.PermissionProviderImpl.getTreePermission(PermissionProviderImpl.java:151)
at
org.apache.jackrabbit.oak.security.authorization.composite.CompositeTreePermission.create(CompositeTreePermission.java:67)
at
org.apache.jackrabbit.oak.security.authorization.composite.CompositePermissionProvider.getTreePermission(CompositePermissionProvider.java:147)
at
org.apache.jackrabbit.oak.core.SecureNodeBuilder.getTreePermission(SecureNodeBuilder.java:357)
at
org.apache.jackrabbit.oak.core.SecureNodeBuilder.getTreePermission(SecureNodeBuilder.java:360)
at
org.apache.jackrabbit.oak.core.SecureNodeBuilder.getTreePermission(SecureNodeBuilder.java:360)
at
org.apache.jackrabbit.oak.core.SecureNodeBuilder.access$100(SecureNodeBuilder.java:49)
at
org.apache.jackrabbit.oak.core.SecureNodeBuilder$ReadablePropertyPredicate.apply(SecureNodeBuilder.java:377)
at
org.apache.jackrabbit.oak.core.SecureNodeBuilder.getProperty(SecureNodeBuilder.java:184)
at
org.apache.jackrabbit.oak.plugins.tree.impl.AbstractTree.getProperty(AbstractTree.java:249)
at org.apache.jackrabbit.oak.core.MutableTree.getProperty(MutableTree.java:128)
at org.apache.jackrabbit.oak.util.TreeUtil.getStringInternal(TreeUtil.java:108)
at org.apache.jackrabbit.oak.util.TreeUtil.getString(TreeUtil.java:101)
at
org.apache.jackrabbit.oak.namepath.GlobalNameMapper.getNamespacesProperty(GlobalNameMapper.java:191)
at
org.apache.jackrabbit.oak.namepath.GlobalNameMapper.getOakURIOrNull(GlobalNameMapper.java:187)
at
org.apache.jackrabbit.oak.jcr.session.SessionNamespaces.getNamespaceURI(SessionNamespaces.java:128)
- locked <0x73c7f51d> (a
org.apache.jackrabbit.oak.jcr.session.SessionNamespaces)
at
org.apache.jackrabbit.oak.jcr.session.SessionImpl.getNamespaceURI(SessionImpl.java:738)
at
com.adobe.granite.repository.impl.CRX3SessionImpl.getNamespaceURI(CRX3SessionImpl.java:283)
at
org.apache.sling.resourceresolver.impl.JcrNamespaceMangler.unmangleNamespaces(JcrNamespaceMangler.java:97)
at
org.apache.sling.resourceresolver.impl.ResourceResolverImpl.unmangleNamespaces(ResourceResolverImpl.java:1109)
at
org.apache.sling.resourceresolver.impl.ResourceResolverImpl.resolveInternal(ResourceResolverImpl.java:263)
at
org.apache.sling.resourceresolver.impl.ResourceResolverImpl.resolve(ResourceResolverImpl.java:250)
at
org.apache.sling.engine.impl.request.RequestData.initResource(RequestData.java:252)
at
org.apache.sling.engine.impl.SlingRequestProcessorImpl.doProcessRequest(SlingRequestProcessorImpl.java:140)
at
org.apache.sling.engine.impl.SlingMainServlet.service(SlingMainServlet.java:219)
at
org.apache.felix.http.base.internal.handler.ServletHandler.handle(ServletHandler.java:85)
at
org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:79)
at
com.adobe.granite.license.impl.LicenseCheckFilter.doFilter(LicenseCheckFilter.java:308)
at
org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
at
org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
at
org.apache.felix.http.sslfilter.internal.SslFilter.doFilter(SslFilter.java:96)
at
org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
at
org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
at
org.apache.sling.security.impl.ReferrerFilter.doFilter(ReferrerFilter.java:295)
at
org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
at
org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
at org.apache.sling.i18n.impl.I18NFilter.doFilter(I18NFilter.java:138)
at
org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
at
org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
at
org.apache.sling.featureflags.impl.FeatureManager.doFilter(FeatureManager.java:116)
at
org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
at
org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
at
org.apache.sling.engine.impl.log.RequestLoggerFilter.doFilter(RequestLoggerFilter.java:72)
at
org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
at
org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
at
org.apache.felix.http.base.internal.dispatch.Dispatcher.dispatch(Dispatcher.java:128)
at
org.apache.felix.http.base.internal.dispatch.DispatcherServlet.service(DispatcherServlet.java:49)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:725)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:215)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
at org.eclipse.jetty.server.Server.handle(Server.java:499)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
at java.lang.Thread.run(Thread.java:748)
{noformat}
> Threads are going in blocked state - OAK segment
> ------------------------------------------------
>
> Key: OAK-7194
> URL: https://issues.apache.org/jira/browse/OAK-7194
> Project: Jackrabbit Oak
> Issue Type: Bug
> Components: segment-tar
> Affects Versions: 1.6.2
> Reporter: Kshitiz Garg
> Priority: Critical
>
> We are using AEM and it internally is using OAK 1.6.2. Our application is
> going to a choked state many a times. Our thread dump analysis is always
> pointing to this stack trace with many blocked threads. Is it a known issue?
> Or is there a setting to avoid it? We are using tar files based repository
> underneath and are using default settings:
>
> {noformat}
> - nativeId:0x1208 - state:BLOCKED
> stackTrace:
> java.lang.Thread.State: BLOCKED (on object monitor)
> at org.apache.jackrabbit.oak.segment.SegmentId.getSegment(SegmentId.java:121)
> - waiting to lock <0x0000000414d8c250> (a
> org.apache.jackrabbit.oak.segment.SegmentId)
> at org.apache.jackrabbit.oak.segment.Record.getSegment(Record.java:70)
> at org.apache.jackrabbit.oak.segment.MapRecord.getEntry(MapRecord.java:160)
> at org.apache.jackrabbit.oak.segment.MapRecord.getEntry(MapRecord.java:192)
> at org.apache.jackrabbit.oak.segment.MapRecord.getEntry(MapRecord.java:192)
> at org.apache.jackrabbit.oak.segment.MapRecord.getEntry(MapRecord.java:192)
> at
> org.apache.jackrabbit.oak.segment.SegmentNodeState.getChildNode(SegmentNodeState.java:412)
> at
> org.apache.jackrabbit.oak.plugins.tree.impl.ImmutableTree.createChild(ImmutableTree.java:125)
> at
> org.apache.jackrabbit.oak.plugins.tree.impl.ImmutableTree.getChild(ImmutableTree.java:176)
> at
> org.apache.jackrabbit.oak.plugins.tree.impl.ImmutableTree.getChild(ImmutableTree.java:81)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.PermissionUtil.getPrincipalRoot(PermissionUtil.java:83)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.PermissionStoreImpl.getPrincipalRoot(PermissionStoreImpl.java:132)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.PermissionStoreImpl.getNumEntries(PermissionStoreImpl.java:103)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.PermissionEntryCache.getNumEntries(PermissionEntryCache.java:102)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.PermissionEntryProviderImpl.init(PermissionEntryProviderImpl.java:79)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.PermissionEntryProviderImpl.<init>(PermissionEntryProviderImpl.java:72)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.CompiledPermissionImpl.<init>(CompiledPermissionImpl.java:112)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.CompiledPermissionImpl.create(CompiledPermissionImpl.java:126)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.PermissionProviderImpl.getCompiledPermissions(PermissionProviderImpl.java:162)
> at
> org.apache.jackrabbit.oak.security.authorization.permission.PermissionProviderImpl.getTreePermission(PermissionProviderImpl.java:151)
> at
> org.apache.jackrabbit.oak.security.authorization.composite.CompositeTreePermission.create(CompositeTreePermission.java:67)
> at
> org.apache.jackrabbit.oak.security.authorization.composite.CompositePermissionProvider.getTreePermission(CompositePermissionProvider.java:147)
> at
> org.apache.jackrabbit.oak.core.SecureNodeBuilder.getTreePermission(SecureNodeBuilder.java:357)
> at
> org.apache.jackrabbit.oak.core.SecureNodeBuilder.getTreePermission(SecureNodeBuilder.java:360)
> at
> org.apache.jackrabbit.oak.core.SecureNodeBuilder.getTreePermission(SecureNodeBuilder.java:360)
> at
> org.apache.jackrabbit.oak.core.SecureNodeBuilder.access$100(SecureNodeBuilder.java:49)
> at
> org.apache.jackrabbit.oak.core.SecureNodeBuilder$ReadablePropertyPredicate.apply(SecureNodeBuilder.java:377)
> at
> org.apache.jackrabbit.oak.core.SecureNodeBuilder.getProperty(SecureNodeBuilder.java:184)
> at
> org.apache.jackrabbit.oak.plugins.tree.impl.AbstractTree.getProperty(AbstractTree.java:249)
> at
> org.apache.jackrabbit.oak.core.MutableTree.getProperty(MutableTree.java:128)
> at
> org.apache.jackrabbit.oak.util.TreeUtil.getStringInternal(TreeUtil.java:108)
> at org.apache.jackrabbit.oak.util.TreeUtil.getString(TreeUtil.java:101)
> at
> org.apache.jackrabbit.oak.namepath.GlobalNameMapper.getNamespacesProperty(GlobalNameMapper.java:191)
> at
> org.apache.jackrabbit.oak.namepath.GlobalNameMapper.getOakURIOrNull(GlobalNameMapper.java:187)
> - eliminated <0x000000070c692f50> (a
> org.apache.jackrabbit.oak.jcr.session.SessionNamespaces)
> at
> org.apache.jackrabbit.oak.jcr.session.SessionNamespaces.getNamespaceURI(SessionNamespaces.java:128)
> - locked <0x000000070c692f50> (a
> org.apache.jackrabbit.oak.jcr.session.SessionNamespaces)
> at
> org.apache.jackrabbit.oak.jcr.session.SessionImpl.getNamespaceURI(SessionImpl.java:738)
> at
> com.adobe.granite.repository.impl.CRX3SessionImpl.getNamespaceURI(CRX3SessionImpl.java:283)
> at
> org.apache.sling.resourceresolver.impl.JcrNamespaceMangler.unmangleNamespaces(JcrNamespaceMangler.java:97)
> at
> org.apache.sling.resourceresolver.impl.ResourceResolverImpl.unmangleNamespaces(ResourceResolverImpl.java:1109)
> at
> org.apache.sling.resourceresolver.impl.ResourceResolverImpl.resolveInternal(ResourceResolverImpl.java:263)
> at
> org.apache.sling.resourceresolver.impl.ResourceResolverImpl.resolve(ResourceResolverImpl.java:250)
> at
> org.apache.sling.engine.impl.request.RequestData.initResource(RequestData.java:252)
> at
> org.apache.sling.engine.impl.SlingRequestProcessorImpl.doProcessRequest(SlingRequestProcessorImpl.java:140)
> at
> org.apache.sling.engine.impl.SlingMainServlet.service(SlingMainServlet.java:219)
> at
> org.apache.felix.http.base.internal.handler.ServletHandler.handle(ServletHandler.java:85)
> at
> org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:79)
> at
> com.adobe.granite.license.impl.LicenseCheckFilter.doFilter(LicenseCheckFilter.java:308)
> at
> org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
> at
> org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
> at
> org.apache.felix.http.sslfilter.internal.SslFilter.doFilter(SslFilter.java:96)
> at
> org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
> at
> org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
> at
> org.apache.sling.security.impl.ReferrerFilter.doFilter(ReferrerFilter.java:295)
> at
> org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
> at
> org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
> at org.apache.sling.i18n.impl.I18NFilter.doFilter(I18NFilter.java:138)
> at
> org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
> at
> org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
> at
> org.apache.sling.featureflags.impl.FeatureManager.doFilter(FeatureManager.java:116)
> at
> org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
> at
> org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
> at
> org.apache.sling.engine.impl.log.RequestLoggerFilter.doFilter(RequestLoggerFilter.java:72)
> at
> org.apache.felix.http.base.internal.handler.FilterHandler.handle(FilterHandler.java:135)
> at
> org.apache.felix.http.base.internal.dispatch.InvocationChain.doFilter(InvocationChain.java:74)
> at
> org.apache.felix.http.base.internal.dispatch.Dispatcher.dispatch(Dispatcher.java:128)
> at
> org.apache.felix.http.base.internal.dispatch.DispatcherServlet.service(DispatcherServlet.java:49)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:725)
> at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:812)
> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587)
> at
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127)
> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
> at
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:215)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
> at org.eclipse.jetty.server.Server.handle(Server.java:499)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311)
> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
> at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
> at java.lang.Thread.run(Thread.java:748)
> Locked ownable synchronizers:
> - None
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
