[
https://issues.apache.org/jira/browse/SOLR-15420?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17348211#comment-17348211
]
Florin Babes commented on SOLR-15420:
-------------------------------------
Hello [~janhoy] ,
Yes, I am doing atomic updates of just one field.
java --version
openjdk 11.0.5 2019-10-15 LTS
OpenJDK Runtime Environment 18.9 (build 11.0.5+10-LTS)
OpenJDK 64-Bit Server VM 18.9 (build 11.0.5+10-LTS, mixed mode, sharing)
Request handler is this
{code:java}
<updateRequestProcessorChain name="tolerant-chain">
<processor class="solr.TolerantUpdateProcessorFactory">
<int name="maxErrors">-1</int>
</processor>
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
<requestHandler name="/update-bulk" class="solr.UpdateRequestHandler">
<lst name="defaults">
<str name = "update.chain">tolerant-chain</str>
</lst>
</requestHandler>
{code}
The schema looks like this:
{code:java}
<?xml version="1.0" encoding="UTF-8"?>
<schema name="example-data-driven-schema" version="1.6">
<uniqueKey>product_id</uniqueKey>
<similarity class="solr.SchemaSimilarityFactory" />
<fieldType name="plong" class="solr.LongPointField" />
<fieldType name="tags" class="solr.TextField" omitNorms="true"
positionIncrementGap="100" enableGraphQueries="false"
autoGeneratePhraseQueries="false">
<analyzer type="index">
<tokenizer class="solr.PatternTokenizerFactory"
pattern="([a-zA-Z0-9]+|\s+|\|\d+\.\d+)" group="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.HunspellStemFilterFactory" dictionary="hunspell_ro.dic"
affix="hunspell_ro.aff" ignoreCase="true" strictAffixParsing="true" />
</analyzer>
<analyzer type="query">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.HunspellStemFilterFactory" dictionary="hunspell_ro.dic"
affix="hunspell_ro.aff" ignoreCase="true" strictAffixParsing="true" />
</analyzer>
</fieldType>
<field name="_version_" type="plong" indexed="true" stored="true"
docValues="true" />
<field name="id" type="string" indexed="true" stored="true" required="true"
docValues="true" />
<field name="product_id" type="string" indexed="true" stored="true"
required="true" docValues="true" />
<field name="tags" type="tags" stored="true" indexed="true" multiValued="true"
termVectors="true" />
</schema>
{code}
This is an update request but instead of 2 docs we send 1000 per request
{code:java}
POST /solr/core_name/update-bulk/?version=2.2&indent=on&wt=xml HTTP/1.1
Host: localhost:8983
Accept: */*
Accept-Encoding: deflate, gzip
Accept-Charset: utf-8
Keep-Alive: 300
Connection: keep-alive
Content-Type: text/xml;charset=UTF-8
Content-Length: 9255
<?xml version="1.0" encoding="UTF-8"?>
<add overwrite="true">
<doc>
<field name="id">73615270</field>
<field name="product_id">37764204</field>
<field name="_version_">1</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags_product_traffic" update="removeregex">.*</field>
</doc>
<doc>
<field name="id">73615327</field>
<field name="product_id">37764210</field>
<field name="_version_">1</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
<field name="tags" update="set">word1 word2</field>
</doc>
</add>
{code}
> Exception thrown during intensive batch partial updating and segment merging
> ----------------------------------------------------------------------------
>
> Key: SOLR-15420
> URL: https://issues.apache.org/jira/browse/SOLR-15420
> Project: Solr
> Issue Type: Bug
> Security Level: Public(Default Security Level. Issues are Public)
> Affects Versions: 8.8.2
> Reporter: Florin Babes
> Priority: Major
>
> Hello,
> I have the following situation:
> solr 8.8.2 standalone
> core with ~7 million documents
> Daily we run an import in which we update all 7 million documents. The
> updated field is a multifield that can contain up to 300 string (max 5
> words). Updating so many documents triggers a segment merge in which big
> segments are merged (~5-6GB). The merge never finishes and solr raises the
> following exception. Also documents are written in tlog.
> I've tested also with 60 string per field, same thing happens.
> It this a bug or it can be a misconfiguration?I've tried to follow the
> stacktrace but I can't find a failure point.
> 2021-05-10 00:31:50.145 ERROR (qtp507819576-40899) [ x:products]
> o.a.s.s.HttpSolrCall org.apache.solr.common.SolrException: Server error
> writing document id 3664065 to the index
> at
> org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:246)
> at
> org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:73)
> at
> org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:256)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:495)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:336)
> at
> org.apache.solr.update.VersionBucket.runWithLock(VersionBucket.java:50)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:336)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:222)
> at
> org.apache.solr.handler.loader.XMLLoader.processUpdate(XMLLoader.java:263)
> at org.apache.solr.handler.loader.XMLLoader.load(XMLLoader.java:190)
> at
> org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
> at
> org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:82)
> at
> org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:216)
> at org.apache.solr.core.SolrCore.execute(SolrCore.java:2646)
> at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:794)
> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:567)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
> at
> org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:357)
> at
> org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:201)
> at
> org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
> at
> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
> at
> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:602)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
> at
> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1612)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1582)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
> at
> org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:191)
> at
> org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177)
> at
> org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
> at
> org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
> at org.eclipse.jetty.server.Server.handle(Server.java:516)
> at
> org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
> at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:556)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273)
> at
> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
> at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
> at
> org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129)
> at
> org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:375)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905)
> at java.base/java.lang.Thread.run(Thread.java:834)
> Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter
> is closed
> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:877)
> at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:891)
> at
> org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1468)
> at
> org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1464)
> at
> org.apache.solr.update.DirectUpdateHandler2.updateDocOrDocValues(DirectUpdateHandler2.java:967)
> at
> org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:342)
> at
> org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:294)
> at
> org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:241)
> ... 56 more
> Caused by: java.lang.ArithmeticException: integer overflow
> at java.base/java.lang.Math.toIntExact(Math.java:1071)
> at
> org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:166)
> at
> org.apache.lucene.index.StoredFieldsConsumer.finishDocument(StoredFieldsConsumer.java:68)
> at
> org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:460)
> at
> org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:496)
> at
> org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:208)
> at
> org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:415)
> at
> org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1471)
> at
> org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1464)
> at
> org.apache.solr.update.DirectUpdateHandler2.updateDocOrDocValues(DirectUpdateHandler2.java:967)
> at
> org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:342)
> at
> org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:294)
> at
> org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:241)
> at
> org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:73)
> at
> org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:256)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:495)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:336)
> at
> org.apache.solr.update.VersionBucket.runWithLock(VersionBucket.java:50)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:336)
> at
> org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:222)
> at
> org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:55)
> at
> org.apache.solr.update.processor.TolerantUpdateProcessor.processAdd(TolerantUpdateProcessor.java:150)
> ... 48 more
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
