[
https://issues.apache.org/jira/browse/SOLR-11625?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Nikolay Martynov updated SOLR-11625:
------------------------------------
Description:
This has been observed in the wild:
{noformat}
2017-11-07 02:35:46.909 ERROR (qtp1724399560-8090) [c:xxx s:shard4
r:core_node399 x:xxx_shard4_replica8] o.a.s.c.SolrCore
:java.nio.channels.ClosedByInterruptException
at
java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
at sun.nio.ch.FileChannelImpl.size(FileChannelImpl.java:315)
at
org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242)
at
org.apache.lucene.store.NRTCachingDirectory.openInput(NRTCachingDirectory.java:192)
at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:356)
at
org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3044)
at org.apache.solr.core.SolrCore.close(SolrCore.java:1575)
at org.apache.solr.servlet.HttpSolrCall.destroy(HttpSolrCall.java:582)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:374)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at
org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
2017-11-07 02:35:46.912 INFO
(OldIndexDirectoryCleanupThreadForCore-xxx_shard4_replica8) [c:xxx s:shard4
r:core_node399 x:xxx_shard4_replica8] o.a.s.c.DirectoryFactory Found 1 old
index directories to clean-up under
/opt/solr/server/solr/xxx_shard4_replica8/data/ afterReload=false
{noformat}
After this Solr cannot start claiming that some files that are supposed to
exist in the index do not exist. On one occasion we observed segments file not
being present.
We were able to trace this problem to {{SolrCore.cleanupOldIndexDirectories}}
using wrong index directory as current index because
{{SolrCore.getNewIndexDir}} could not read proper index directory due to
reading code receiving interruption exception.
[This
change|https://github.com/mar-kolya/lucene-solr/commit/8967367edd2b8b5ed072876f27051613e3425100]
seems to address the problem. But it should be said that this is more of a
hot-patch rather than a proper fix.
was:
This has been observed in the wild:
{noformat}
2017-11-07 02:35:46.909 ERROR (qtp1724399560-8090) [c:xxx s:shard4
r:core_node399 x:xxx_shard4_replica8] o.a.s.c.SolrCore
:java.nio.channels.ClosedByInterruptException
at
java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
at sun.nio.ch.FileChannelImpl.size(FileChannelImpl.java:315)
at
org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242)
at
org.apache.lucene.store.NRTCachingDirectory.openInput(NRTCachingDirectory.java:192)
at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:356)
at
org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3044)
at org.apache.solr.core.SolrCore.close(SolrCore.java:1575)
at org.apache.solr.servlet.HttpSolrCall.destroy(HttpSolrCall.java:582)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:374)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at
org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:534)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at
org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at
org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
2017-11-07 02:35:46.912 INFO
(OldIndexDirectoryCleanupThreadForCore-xxx_shard4_replica8) [c:xxx s:shard4
r:core_node399 x:xxx_shard4_replica8] o.a.s.c.DirectoryFactory Found 1 old
index directories to clean-up under
/opt/solr/server/solr/xxx_shard4_replica8/data/ afterReload=false
{noformat}
After this Solr cannot start claiming that some files that are supposed to
exist in the index do not exist. On one occasion we observed segments file not
being present.
We were able to trace this problem to {SolrCore.cleanupOldIndexDirectories}
using wrong index directory as current index because {SolrCore.getNewIndexDir}
could not read proper index directory due to reading code receiving
interruption exception.
[This
change|https://github.com/mar-kolya/lucene-solr/commit/8967367edd2b8b5ed072876f27051613e3425100]
seems to address the problem. But it should be said that this is more of a
hot-patch rather than a proper fix.
> Solr may remove live index on Solr shutdown
> -------------------------------------------
>
> Key: SOLR-11625
> URL: https://issues.apache.org/jira/browse/SOLR-11625
> Project: Solr
> Issue Type: Bug
> Security Level: Public(Default Security Level. Issues are Public)
> Affects Versions: 6.6.1
> Reporter: Nikolay Martynov
>
> This has been observed in the wild:
> {noformat}
> 2017-11-07 02:35:46.909 ERROR (qtp1724399560-8090) [c:xxx s:shard4
> r:core_node399 x:xxx_shard4_replica8] o.a.s.c.SolrCore
> :java.nio.channels.ClosedByInterruptException
> at
> java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
> at sun.nio.ch.FileChannelImpl.size(FileChannelImpl.java:315)
> at
> org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242)
> at
> org.apache.lucene.store.NRTCachingDirectory.openInput(NRTCachingDirectory.java:192)
> at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:356)
> at
> org.apache.solr.core.SolrCore.cleanupOldIndexDirectories(SolrCore.java:3044)
> at org.apache.solr.core.SolrCore.close(SolrCore.java:1575)
> at org.apache.solr.servlet.HttpSolrCall.destroy(HttpSolrCall.java:582)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:374)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
> at
> org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1691)
> at
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
> at
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
> at
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
> at
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
> at
> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
> at
> org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
> at org.eclipse.jetty.server.Server.handle(Server.java:534)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
> at
> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
> at
> org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
> at
> org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
> at
> org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
> at
> org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
> at java.lang.Thread.run(Thread.java:748)
> 2017-11-07 02:35:46.912 INFO
> (OldIndexDirectoryCleanupThreadForCore-xxx_shard4_replica8) [c:xxx s:shard4
> r:core_node399 x:xxx_shard4_replica8] o.a.s.c.DirectoryFactory Found 1 old
> index directories to clean-up under
> /opt/solr/server/solr/xxx_shard4_replica8/data/ afterReload=false
> {noformat}
> After this Solr cannot start claiming that some files that are supposed to
> exist in the index do not exist. On one occasion we observed segments file
> not being present.
> We were able to trace this problem to {{SolrCore.cleanupOldIndexDirectories}}
> using wrong index directory as current index because
> {{SolrCore.getNewIndexDir}} could not read proper index directory due to
> reading code receiving interruption exception.
> [This
> change|https://github.com/mar-kolya/lucene-solr/commit/8967367edd2b8b5ed072876f27051613e3425100]
> seems to address the problem. But it should be said that this is more of a
> hot-patch rather than a proper fix.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
