woah, def not - bad merge of somehow, thanks. - Mark
On Feb 1, 2013, at 11:12 AM, Alan Woodward <[email protected]> wrote: > Hey Mark, did you mean to back out these CHANGES.TXT entries? > > Alan Woodward > www.flax.co.uk > > > On 1 Feb 2013, at 15:29, [email protected] wrote: > >> Author: markrmiller >> Date: Fri Feb 1 15:29:08 2013 >> New Revision: 1441483 >> >> URL: http://svn.apache.org/viewvc?rev=1441483&view=rev >> Log: >> SOLR-4370: Allow configuring commitWithin to do hard commits. >> >> Added: >> >> lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java >> (with props) >> Modified: >> lucene/dev/trunk/solr/CHANGES.txt >> lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java >> >> lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java >> >> lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml >> >> lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java >> >> Modified: lucene/dev/trunk/solr/CHANGES.txt >> URL: >> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1441483&r1=1441482&r2=1441483&view=diff >> ============================================================================== >> --- lucene/dev/trunk/solr/CHANGES.txt (original) >> +++ lucene/dev/trunk/solr/CHANGES.txt Fri Feb 1 15:29:08 2013 >> @@ -68,6 +68,9 @@ New Features >> >> * SOLR-2827: RegexpBoost Update Processor (janhoy) >> >> +* SOLR-4370: Allow configuring commitWithin to do hard commits. >> + (Mark Miller, Senthuran Sivananthan) >> + >> Bug Fixes >> ---------------------- >> >> @@ -97,11 +100,6 @@ Bug Fixes >> >> * SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman) >> >> -* SOLR-3967: langid.enforceSchema option checks source field instead of >> target field (janhoy) >> - >> -* SOLR-4380: Replicate after startup option would not replicate until the >> - IndexWriter was lazily opened. (Mark Miller, Gregg Donovan) >> - >> Optimizations >> ---------------------- >> >> @@ -119,11 +117,6 @@ Optimizations >> * SOLR-4306: Utilize indexInfo=false when gathering core names in UI >> (steffkes) >> >> -* SOLR-4284: Admin UI - make core list scrollable separate from the rest of >> - the UI (steffkes) >> - >> -* SOLR-4364: Admin UI - Locale based number formatting (steffkes) >> - >> Other Changes >> ---------------------- >> >> @@ -132,8 +125,6 @@ Other Changes >> >> * SOLR-4353: Renamed example jetty context file to reduce confusion (hossman) >> >> -* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy) >> - >> ================== 4.1.0 ================== >> >> Versions of Major Components >> >> Modified: >> lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java >> URL: >> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1441483&r1=1441482&r2=1441483&view=diff >> ============================================================================== >> --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java >> (original) >> +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java >> Fri Feb 1 15:29:08 2013 >> @@ -238,7 +238,8 @@ public class SolrConfig extends Config { >> getBool("updateHandler/autoCommit/openSearcher",true), >> getInt("updateHandler/commitIntervalLowerBound",-1), >> getInt("updateHandler/autoSoftCommit/maxDocs",-1), >> - getInt("updateHandler/autoSoftCommit/maxTime",-1)); >> + getInt("updateHandler/autoSoftCommit/maxTime",-1), >> + getBool("updateHandler/commitWithin/softCommit",true)); >> } >> >> private void loadPluginInfo(Class clazz, String tag, boolean requireName, >> boolean requireClass) { >> @@ -402,6 +403,7 @@ public class SolrConfig extends Config { >> public final int >> autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound, >> autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime; >> public final boolean openSearcher; // is opening a new searcher part of >> hard autocommit? >> + public final boolean commitWithinSoftCommit; >> >> /** >> * @param autoCommmitMaxDocs set -1 as default >> @@ -409,7 +411,7 @@ public class SolrConfig extends Config { >> * @param commitIntervalLowerBound set -1 as default >> */ >> public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int >> autoCommmitMaxTime, boolean openSearcher, int commitIntervalLowerBound, >> - int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime) { >> + int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean >> commitWithinSoftCommit) { >> this.className = className; >> this.autoCommmitMaxDocs = autoCommmitMaxDocs; >> this.autoCommmitMaxTime = autoCommmitMaxTime; >> @@ -418,6 +420,8 @@ public class SolrConfig extends Config { >> >> this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs; >> this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime; >> + >> + this.commitWithinSoftCommit = commitWithinSoftCommit; >> } >> } >> >> >> Modified: >> lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java >> URL: >> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1441483&r1=1441482&r2=1441483&view=diff >> ============================================================================== >> --- >> lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java >> (original) >> +++ >> lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java >> Fri Feb 1 15:29:08 2013 >> @@ -91,6 +91,8 @@ public class DirectUpdateHandler2 extend >> // tracks when auto-commit should occur >> protected final CommitTracker commitTracker; >> protected final CommitTracker softCommitTracker; >> + >> + protected boolean commitWithinSoftCommit; >> >> public DirectUpdateHandler2(SolrCore core) { >> super(core); >> @@ -106,6 +108,8 @@ public class DirectUpdateHandler2 extend >> int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; >> // getInt("updateHandler/autoSoftCommit/maxDocs", -1); >> int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; >> // getInt("updateHandler/autoSoftCommit/maxTime", -1); >> softCommitTracker = new CommitTracker("Soft", core, >> softCommitDocsUpperBound, softCommitTimeUpperBound, true, true); >> + >> + commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit; >> } >> >> public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) { >> @@ -126,6 +130,8 @@ public class DirectUpdateHandler2 extend >> if (this.ulog != null) { >> this.ulog.init(this, core); >> } >> + >> + commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit; >> } >> >> private void deleteAll() throws IOException { >> @@ -229,8 +235,13 @@ public class DirectUpdateHandler2 extend >> } >> >> if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) { >> - commitTracker.addedDocument(-1); >> - softCommitTracker.addedDocument(cmd.commitWithin); >> + if (commitWithinSoftCommit) { >> + commitTracker.addedDocument(-1); >> + softCommitTracker.addedDocument(cmd.commitWithin); >> + } else { >> + softCommitTracker.addedDocument(-1); >> + commitTracker.addedDocument(cmd.commitWithin); >> + } >> } >> >> rc = 1; >> @@ -252,7 +263,11 @@ public class DirectUpdateHandler2 extend >> >> private void updateDeleteTrackers(DeleteUpdateCommand cmd) { >> if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) { >> - softCommitTracker.deletedDocument(cmd.commitWithin); >> + if (commitWithinSoftCommit) { >> + softCommitTracker.deletedDocument(cmd.commitWithin); >> + } else { >> + commitTracker.deletedDocument(cmd.commitWithin); >> + } >> >> if (commitTracker.getTimeUpperBound() > 0) { >> >> commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound()); >> >> Modified: >> lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml >> URL: >> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1441483&r1=1441482&r2=1441483&view=diff >> ============================================================================== >> --- >> lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml >> (original) >> +++ >> lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml >> Fri Feb 1 15:29:08 2013 >> @@ -97,6 +97,10 @@ >> <updateLog enable="${enable.update.log:false}"> >> <str name="dir">${solr.ulog.dir:}</str> >> </updateLog> >> + >> + <commitWithin> >> + <softCommit>${solr.commitwithin.softcommit:true}</softCommit> >> + </commitWithin> >> >> </updateHandler> >> >> >> Modified: >> lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java >> URL: >> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java?rev=1441483&r1=1441482&r2=1441483&view=diff >> ============================================================================== >> --- >> lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java >> (original) >> +++ >> lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java >> Fri Feb 1 15:29:08 2013 >> @@ -286,7 +286,7 @@ public class AutoCommitTest extends Abst >> >> MapSolrParams params = new MapSolrParams( new HashMap<String, String>() >> ); >> >> - // Add a single document with commitWithin == 1 second >> + // Add a single document with commitWithin == 2 second >> SolrQueryResponse rsp = new SolrQueryResponse(); >> SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {}; >> req.setContentStreams( toContentStreams( >> >> Added: >> lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java >> URL: >> http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java?rev=1441483&view=auto >> ============================================================================== >> --- >> lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java >> (added) >> +++ >> lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java >> Fri Feb 1 15:29:08 2013 >> @@ -0,0 +1,136 @@ >> +/* >> + * Licensed to the Apache Software Foundation (ASF) under one or more >> + * contributor license agreements. See the NOTICE file distributed with >> + * this work for additional information regarding copyright ownership. >> + * The ASF licenses this file to You under the Apache License, Version 2.0 >> + * (the "License"); you may not use this file except in compliance with >> + * the License. You may obtain a copy of the License at >> + * >> + * http://www.apache.org/licenses/LICENSE-2.0 >> + * >> + * Unless required by applicable law or agreed to in writing, software >> + * distributed under the License is distributed on an "AS IS" BASIS, >> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. >> + * See the License for the specific language governing permissions and >> + * limitations under the License. >> + */ >> + >> +package org.apache.solr.update; >> + >> +import java.util.HashMap; >> + >> +import org.apache.lucene.util.LuceneTestCase.Slow; >> +import org.apache.solr.common.params.MapSolrParams; >> +import org.apache.solr.core.SolrCore; >> +import org.apache.solr.handler.UpdateRequestHandler; >> +import org.apache.solr.request.SolrQueryRequestBase; >> +import org.apache.solr.response.SolrQueryResponse; >> +import org.apache.solr.util.AbstractSolrTestCase; >> +import org.junit.AfterClass; >> +import org.junit.BeforeClass; >> + >> +@Slow >> +public class HardAutoCommitTest extends AbstractSolrTestCase { >> + >> + @BeforeClass >> + public static void beforeClass() throws Exception { >> + System.setProperty("solr.commitwithin.softcommit", "false"); >> + initCore("solrconfig.xml", "schema.xml"); >> + } >> + >> + @AfterClass >> + public static void afterClass() { >> + System.clearProperty("solr.commitwithin.softcommit"); >> + } >> + >> + @Override >> + public void setUp() throws Exception { >> + super.setUp(); >> + clearIndex(); >> + // reload the core to clear stats >> + h.getCoreContainer().reload(h.getCore().getName()); >> + } >> + >> + >> + public void testCommitWithin() throws Exception { >> + SolrCore core = h.getCore(); >> + >> + NewSearcherListener trigger = new NewSearcherListener(); >> + core.registerNewSearcherListener(trigger); >> + DirectUpdateHandler2 updater = (DirectUpdateHandler2) >> core.getUpdateHandler(); >> + CommitTracker tracker = updater.commitTracker; >> + tracker.setTimeUpperBound(0); >> + tracker.setDocsUpperBound(-1); >> + >> + UpdateRequestHandler handler = new UpdateRequestHandler(); >> + handler.init( null ); >> + >> + MapSolrParams params = new MapSolrParams( new HashMap<String, String>() >> ); >> + >> + // Add a single document with commitWithin == 2 second >> + SolrQueryResponse rsp = new SolrQueryResponse(); >> + SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {}; >> + req.setContentStreams( AutoCommitTest.toContentStreams( >> + adoc(2000, "id", "529", "field_t", "what's inside?", "subject", >> "info"), null ) ); >> + trigger.reset(); >> + handler.handleRequest( req, rsp ); >> + >> + // Check it isn't in the index >> + assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" ); >> + >> + // Wait longer than the commitWithin time >> + assertTrue("commitWithin failed to commit", >> trigger.waitForNewSearcher(30000)); >> + >> + // Add one document without commitWithin >> + req.setContentStreams( AutoCommitTest.toContentStreams( >> + adoc("id", "530", "field_t", "what's inside?", "subject", "info"), >> null ) ); >> + trigger.reset(); >> + handler.handleRequest( req, rsp ); >> + >> + // Check it isn't in the index >> + assertQ("shouldn't find any", req("id:530") ,"//result[@numFound=0]" ); >> + >> + // Delete one document with commitWithin >> + req.setContentStreams( AutoCommitTest.toContentStreams( >> + delI("529", "commitWithin", "1000"), null ) ); >> + trigger.reset(); >> + handler.handleRequest( req, rsp ); >> + >> + // Now make sure we can find it >> + assertQ("should find one", req("id:529") ,"//result[@numFound=1]" ); >> + >> + // Wait for the commit to happen >> + assertTrue("commitWithin failed to commit", >> trigger.waitForNewSearcher(30000)); >> + >> + // Now we shouldn't find it >> + assertQ("should find none", req("id:529") ,"//result[@numFound=0]" ); >> + // ... but we should find the new one >> + assertQ("should find one", req("id:530") ,"//result[@numFound=1]" ); >> + >> + trigger.reset(); >> + >> + // now make the call 10 times really fast and make sure it >> + // only commits once >> + req.setContentStreams( AutoCommitTest.toContentStreams( >> + adoc(2000, "id", "500" ), null ) ); >> + for( int i=0;i<10; i++ ) { >> + handler.handleRequest( req, rsp ); >> + } >> + assertQ("should not be there yet", req("id:500") >> ,"//result[@numFound=0]" ); >> + >> + // the same for the delete >> + req.setContentStreams( AutoCommitTest.toContentStreams( >> + delI("530", "commitWithin", "1000"), null ) ); >> + for( int i=0;i<10; i++ ) { >> + handler.handleRequest( req, rsp ); >> + } >> + assertQ("should be there", req("id:530") ,"//result[@numFound=1]" ); >> + >> + assertTrue("commitWithin failed to commit", >> trigger.waitForNewSearcher(30000)); >> + assertQ("should be there", req("id:500") ,"//result[@numFound=1]" ); >> + assertQ("should not be there", req("id:530") ,"//result[@numFound=0]" ); >> + >> + assertEquals(3, tracker.getCommitCount()); >> + } >> + >> +} >> >> > --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
