Hey Mark, did you mean to back out these CHANGES.TXT entries? Alan Woodward www.flax.co.uk
On 1 Feb 2013, at 15:29, [email protected] wrote: > Author: markrmiller > Date: Fri Feb 1 15:29:08 2013 > New Revision: 1441483 > > URL: http://svn.apache.org/viewvc?rev=1441483&view=rev > Log: > SOLR-4370: Allow configuring commitWithin to do hard commits. > > Added: > > lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java > (with props) > Modified: > lucene/dev/trunk/solr/CHANGES.txt > lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java > > lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java > > lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml > > lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java > > Modified: lucene/dev/trunk/solr/CHANGES.txt > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1441483&r1=1441482&r2=1441483&view=diff > ============================================================================== > --- lucene/dev/trunk/solr/CHANGES.txt (original) > +++ lucene/dev/trunk/solr/CHANGES.txt Fri Feb 1 15:29:08 2013 > @@ -68,6 +68,9 @@ New Features > > * SOLR-2827: RegexpBoost Update Processor (janhoy) > > +* SOLR-4370: Allow configuring commitWithin to do hard commits. > + (Mark Miller, Senthuran Sivananthan) > + > Bug Fixes > ---------------------- > > @@ -97,11 +100,6 @@ Bug Fixes > > * SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman) > > -* SOLR-3967: langid.enforceSchema option checks source field instead of > target field (janhoy) > - > -* SOLR-4380: Replicate after startup option would not replicate until the > - IndexWriter was lazily opened. (Mark Miller, Gregg Donovan) > - > Optimizations > ---------------------- > > @@ -119,11 +117,6 @@ Optimizations > * SOLR-4306: Utilize indexInfo=false when gathering core names in UI > (steffkes) > > -* SOLR-4284: Admin UI - make core list scrollable separate from the rest of > - the UI (steffkes) > - > -* SOLR-4364: Admin UI - Locale based number formatting (steffkes) > - > Other Changes > ---------------------- > > @@ -132,8 +125,6 @@ Other Changes > > * SOLR-4353: Renamed example jetty context file to reduce confusion (hossman) > > -* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy) > - > ================== 4.1.0 ================== > > Versions of Major Components > > Modified: > lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1441483&r1=1441482&r2=1441483&view=diff > ============================================================================== > --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java > (original) > +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java > Fri Feb 1 15:29:08 2013 > @@ -238,7 +238,8 @@ public class SolrConfig extends Config { > getBool("updateHandler/autoCommit/openSearcher",true), > getInt("updateHandler/commitIntervalLowerBound",-1), > getInt("updateHandler/autoSoftCommit/maxDocs",-1), > - getInt("updateHandler/autoSoftCommit/maxTime",-1)); > + getInt("updateHandler/autoSoftCommit/maxTime",-1), > + getBool("updateHandler/commitWithin/softCommit",true)); > } > > private void loadPluginInfo(Class clazz, String tag, boolean requireName, > boolean requireClass) { > @@ -402,6 +403,7 @@ public class SolrConfig extends Config { > public final int > autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound, > autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime; > public final boolean openSearcher; // is opening a new searcher part of > hard autocommit? > + public final boolean commitWithinSoftCommit; > > /** > * @param autoCommmitMaxDocs set -1 as default > @@ -409,7 +411,7 @@ public class SolrConfig extends Config { > * @param commitIntervalLowerBound set -1 as default > */ > public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int > autoCommmitMaxTime, boolean openSearcher, int commitIntervalLowerBound, > - int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime) { > + int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean > commitWithinSoftCommit) { > this.className = className; > this.autoCommmitMaxDocs = autoCommmitMaxDocs; > this.autoCommmitMaxTime = autoCommmitMaxTime; > @@ -418,6 +420,8 @@ public class SolrConfig extends Config { > > this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs; > this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime; > + > + this.commitWithinSoftCommit = commitWithinSoftCommit; > } > } > > > Modified: > lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1441483&r1=1441482&r2=1441483&view=diff > ============================================================================== > --- > lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java > (original) > +++ > lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java > Fri Feb 1 15:29:08 2013 > @@ -91,6 +91,8 @@ public class DirectUpdateHandler2 extend > // tracks when auto-commit should occur > protected final CommitTracker commitTracker; > protected final CommitTracker softCommitTracker; > + > + protected boolean commitWithinSoftCommit; > > public DirectUpdateHandler2(SolrCore core) { > super(core); > @@ -106,6 +108,8 @@ public class DirectUpdateHandler2 extend > int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; > // getInt("updateHandler/autoSoftCommit/maxDocs", -1); > int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; > // getInt("updateHandler/autoSoftCommit/maxTime", -1); > softCommitTracker = new CommitTracker("Soft", core, > softCommitDocsUpperBound, softCommitTimeUpperBound, true, true); > + > + commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit; > } > > public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) { > @@ -126,6 +130,8 @@ public class DirectUpdateHandler2 extend > if (this.ulog != null) { > this.ulog.init(this, core); > } > + > + commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit; > } > > private void deleteAll() throws IOException { > @@ -229,8 +235,13 @@ public class DirectUpdateHandler2 extend > } > > if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) { > - commitTracker.addedDocument(-1); > - softCommitTracker.addedDocument(cmd.commitWithin); > + if (commitWithinSoftCommit) { > + commitTracker.addedDocument(-1); > + softCommitTracker.addedDocument(cmd.commitWithin); > + } else { > + softCommitTracker.addedDocument(-1); > + commitTracker.addedDocument(cmd.commitWithin); > + } > } > > rc = 1; > @@ -252,7 +263,11 @@ public class DirectUpdateHandler2 extend > > private void updateDeleteTrackers(DeleteUpdateCommand cmd) { > if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) { > - softCommitTracker.deletedDocument(cmd.commitWithin); > + if (commitWithinSoftCommit) { > + softCommitTracker.deletedDocument(cmd.commitWithin); > + } else { > + commitTracker.deletedDocument(cmd.commitWithin); > + } > > if (commitTracker.getTimeUpperBound() > 0) { > commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound()); > > Modified: > lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1441483&r1=1441482&r2=1441483&view=diff > ============================================================================== > --- > lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml > (original) > +++ > lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml > Fri Feb 1 15:29:08 2013 > @@ -97,6 +97,10 @@ > <updateLog enable="${enable.update.log:false}"> > <str name="dir">${solr.ulog.dir:}</str> > </updateLog> > + > + <commitWithin> > + <softCommit>${solr.commitwithin.softcommit:true}</softCommit> > + </commitWithin> > > </updateHandler> > > > Modified: > lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java?rev=1441483&r1=1441482&r2=1441483&view=diff > ============================================================================== > --- > lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java > (original) > +++ > lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java > Fri Feb 1 15:29:08 2013 > @@ -286,7 +286,7 @@ public class AutoCommitTest extends Abst > > MapSolrParams params = new MapSolrParams( new HashMap<String, String>() ); > > - // Add a single document with commitWithin == 1 second > + // Add a single document with commitWithin == 2 second > SolrQueryResponse rsp = new SolrQueryResponse(); > SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {}; > req.setContentStreams( toContentStreams( > > Added: > lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java > URL: > http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java?rev=1441483&view=auto > ============================================================================== > --- > lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java > (added) > +++ > lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java > Fri Feb 1 15:29:08 2013 > @@ -0,0 +1,136 @@ > +/* > + * Licensed to the Apache Software Foundation (ASF) under one or more > + * contributor license agreements. See the NOTICE file distributed with > + * this work for additional information regarding copyright ownership. > + * The ASF licenses this file to You under the Apache License, Version 2.0 > + * (the "License"); you may not use this file except in compliance with > + * the License. You may obtain a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + * See the License for the specific language governing permissions and > + * limitations under the License. > + */ > + > +package org.apache.solr.update; > + > +import java.util.HashMap; > + > +import org.apache.lucene.util.LuceneTestCase.Slow; > +import org.apache.solr.common.params.MapSolrParams; > +import org.apache.solr.core.SolrCore; > +import org.apache.solr.handler.UpdateRequestHandler; > +import org.apache.solr.request.SolrQueryRequestBase; > +import org.apache.solr.response.SolrQueryResponse; > +import org.apache.solr.util.AbstractSolrTestCase; > +import org.junit.AfterClass; > +import org.junit.BeforeClass; > + > +@Slow > +public class HardAutoCommitTest extends AbstractSolrTestCase { > + > + @BeforeClass > + public static void beforeClass() throws Exception { > + System.setProperty("solr.commitwithin.softcommit", "false"); > + initCore("solrconfig.xml", "schema.xml"); > + } > + > + @AfterClass > + public static void afterClass() { > + System.clearProperty("solr.commitwithin.softcommit"); > + } > + > + @Override > + public void setUp() throws Exception { > + super.setUp(); > + clearIndex(); > + // reload the core to clear stats > + h.getCoreContainer().reload(h.getCore().getName()); > + } > + > + > + public void testCommitWithin() throws Exception { > + SolrCore core = h.getCore(); > + > + NewSearcherListener trigger = new NewSearcherListener(); > + core.registerNewSearcherListener(trigger); > + DirectUpdateHandler2 updater = (DirectUpdateHandler2) > core.getUpdateHandler(); > + CommitTracker tracker = updater.commitTracker; > + tracker.setTimeUpperBound(0); > + tracker.setDocsUpperBound(-1); > + > + UpdateRequestHandler handler = new UpdateRequestHandler(); > + handler.init( null ); > + > + MapSolrParams params = new MapSolrParams( new HashMap<String, String>() > ); > + > + // Add a single document with commitWithin == 2 second > + SolrQueryResponse rsp = new SolrQueryResponse(); > + SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {}; > + req.setContentStreams( AutoCommitTest.toContentStreams( > + adoc(2000, "id", "529", "field_t", "what's inside?", "subject", > "info"), null ) ); > + trigger.reset(); > + handler.handleRequest( req, rsp ); > + > + // Check it isn't in the index > + assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" ); > + > + // Wait longer than the commitWithin time > + assertTrue("commitWithin failed to commit", > trigger.waitForNewSearcher(30000)); > + > + // Add one document without commitWithin > + req.setContentStreams( AutoCommitTest.toContentStreams( > + adoc("id", "530", "field_t", "what's inside?", "subject", "info"), > null ) ); > + trigger.reset(); > + handler.handleRequest( req, rsp ); > + > + // Check it isn't in the index > + assertQ("shouldn't find any", req("id:530") ,"//result[@numFound=0]" ); > + > + // Delete one document with commitWithin > + req.setContentStreams( AutoCommitTest.toContentStreams( > + delI("529", "commitWithin", "1000"), null ) ); > + trigger.reset(); > + handler.handleRequest( req, rsp ); > + > + // Now make sure we can find it > + assertQ("should find one", req("id:529") ,"//result[@numFound=1]" ); > + > + // Wait for the commit to happen > + assertTrue("commitWithin failed to commit", > trigger.waitForNewSearcher(30000)); > + > + // Now we shouldn't find it > + assertQ("should find none", req("id:529") ,"//result[@numFound=0]" ); > + // ... but we should find the new one > + assertQ("should find one", req("id:530") ,"//result[@numFound=1]" ); > + > + trigger.reset(); > + > + // now make the call 10 times really fast and make sure it > + // only commits once > + req.setContentStreams( AutoCommitTest.toContentStreams( > + adoc(2000, "id", "500" ), null ) ); > + for( int i=0;i<10; i++ ) { > + handler.handleRequest( req, rsp ); > + } > + assertQ("should not be there yet", req("id:500") > ,"//result[@numFound=0]" ); > + > + // the same for the delete > + req.setContentStreams( AutoCommitTest.toContentStreams( > + delI("530", "commitWithin", "1000"), null ) ); > + for( int i=0;i<10; i++ ) { > + handler.handleRequest( req, rsp ); > + } > + assertQ("should be there", req("id:530") ,"//result[@numFound=1]" ); > + > + assertTrue("commitWithin failed to commit", > trigger.waitForNewSearcher(30000)); > + assertQ("should be there", req("id:500") ,"//result[@numFound=1]" ); > + assertQ("should not be there", req("id:530") ,"//result[@numFound=0]" ); > + > + assertEquals(3, tracker.getCommitCount()); > + } > + > +} > >
