Author: gsingers
Date: Thu Jun 26 14:14:06 2008
New Revision: 672031
URL: http://svn.apache.org/viewvc?rev=672031&view=rev
Log:
SOLR-603: Add partial optimize capabilities and deprecate DirectUpdateHandler
Added:
lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java
(with props)
lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig-duh-optimize.xml
- copied, changed from r670009,
lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrServer.java
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java
lucene/solr/trunk/src/java/org/apache/solr/common/params/UpdateParams.java
lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
lucene/solr/trunk/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
Modified: lucene/solr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=672031&r1=672030&r2=672031&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Thu Jun 26 14:14:06 2008
@@ -298,6 +298,8 @@
(Shalin Shekhar Mangar, Bojan Smid, gsingers)
54. SOLR-423: Added Request Handler close hook notification so that
RequestHandlers can be notified when a core is closing. (gsingers, ryan)
+
+55. SOLR-603: Added ability to partially optimize. (gsingers)
Changes in runtime behavior
1. SOLR-559: use Lucene updateDocument, deleteDocuments methods. This
Modified:
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrServer.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrServer.java?rev=672031&r1=672030&r2=672031&view=diff
==============================================================================
---
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrServer.java
(original)
+++
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/SolrServer.java
Thu Jun 26 14:14:06 2008
@@ -95,7 +95,7 @@
* @throws IOException
*/
public UpdateResponse optimize( ) throws SolrServerException, IOException {
- return optimize(true, true);
+ return optimize(true, true, 1);
}
public UpdateResponse commit( boolean waitFlush, boolean waitSearcher )
throws SolrServerException, IOException {
@@ -103,7 +103,11 @@
}
public UpdateResponse optimize( boolean waitFlush, boolean waitSearcher )
throws SolrServerException, IOException {
- return new UpdateRequest().setAction( UpdateRequest.ACTION.OPTIMIZE,
waitFlush, waitSearcher ).process( this );
+ return optimize(waitFlush, waitSearcher, 1);
+ }
+
+ public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int
maxSegments ) throws SolrServerException, IOException {
+ return new UpdateRequest().setAction( UpdateRequest.ACTION.OPTIMIZE,
waitFlush, waitSearcher, maxSegments ).process( this );
}
public UpdateResponse deleteById(String id) throws SolrServerException,
IOException {
Modified:
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=672031&r1=672030&r2=672031&view=diff
==============================================================================
---
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java
(original)
+++
lucene/solr/trunk/client/java/solrj/src/org/apache/solr/client/solrj/request/UpdateRequest.java
Thu Jun 26 14:14:06 2008
@@ -127,11 +127,16 @@
/** Sets appropriate parameters for the given ACTION */
public UpdateRequest setAction(ACTION action, boolean waitFlush, boolean
waitSearcher ) {
+ return setAction(action, waitFlush, waitSearcher, 1);
+ }
+
+ public UpdateRequest setAction(ACTION action, boolean waitFlush, boolean
waitSearcher, int maxSegments ) {
if (params == null)
params = new ModifiableSolrParams();
if( action == ACTION.OPTIMIZE ) {
params.set( UpdateParams.OPTIMIZE, "true" );
+ params.set(UpdateParams.MAX_OPTIMIZE_SEGMENTS, maxSegments);
}
else if( action == ACTION.COMMIT ) {
params.set( UpdateParams.COMMIT, "true" );
@@ -140,6 +145,7 @@
params.set( UpdateParams.WAIT_SEARCHER, waitSearcher+"" );
return this;
}
+
public void setParam(String param, String value) {
if (params == null)
Modified:
lucene/solr/trunk/src/java/org/apache/solr/common/params/UpdateParams.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/common/params/UpdateParams.java?rev=672031&r1=672030&r2=672031&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/common/params/UpdateParams.java
(original)
+++ lucene/solr/trunk/src/java/org/apache/solr/common/params/UpdateParams.java
Thu Jun 26 14:14:06 2008
@@ -42,4 +42,8 @@
/** Select the update processor to use. A RequestHandler may or may not
respect this parameter */
public static final String UPDATE_PROCESSOR = "update.processor";
+ /**
+ * If optimizing, set the maximum number of segments left in the index after
optimization. 1 is the default (and is equivalent to calling
IndexWriter.optimize() in Lucene).
+ */
+ public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
}
Modified:
lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=672031&r1=672030&r2=672031&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
(original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
Thu Jun 26 14:14:06 2008
@@ -67,6 +67,7 @@
CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
cmd.waitFlush );
cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
cmd.waitSearcher );
+ cmd.maxOptimizeSegments =
params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
req.getCore().getUpdateHandler().commit( cmd );
// Lets wait till after solr1.2 to define consistent output format
@@ -99,6 +100,7 @@
CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
cmd.waitFlush );
cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
cmd.waitSearcher );
+ cmd.maxOptimizeSegments =
params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
processor.processCommit( cmd );
return true;
}
Modified:
lucene/solr/trunk/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java?rev=672031&r1=672030&r2=672031&view=diff
==============================================================================
---
lucene/solr/trunk/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
(original)
+++
lucene/solr/trunk/src/java/org/apache/solr/handler/XmlUpdateRequestHandler.java
Thu Jun 26 14:14:06 2008
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
+import java.io.File;
import java.util.HashMap;
import java.util.logging.Logger;
@@ -75,7 +76,8 @@
public static final String ALLOW_DUPS = "allowDups";
XMLInputFactory inputFactory;
-
+
+
@Override
public void init(NamedList args)
{
@@ -210,7 +212,10 @@
} else if (WAIT_SEARCHER.equals(attrName)) {
cmd.waitSearcher = StrUtils.parseBoolean(attrVal);
sawWaitSearcher = true;
- } else {
+ } else if (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)){
+ cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
+ }
+ else {
log.warning("unexpected attribute commit/@" + attrName);
}
}
Modified:
lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=672031&r1=672030&r2=672031&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
(original)
+++ lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
Thu Jun 26 14:14:06 2008
@@ -24,6 +24,13 @@
public boolean waitFlush;
public boolean waitSearcher=true;
+ /**
+ * During optimize, optimize down to <= this many segments. Must be >= 1
+ *
+ * @see [EMAIL PROTECTED] org.apache.lucene.index.IndexWriter#optimize(int)}
+ */
+ public int maxOptimizeSegments = 1;
+
public CommitUpdateCommand(boolean optimize) {
super("commit");
this.optimize=optimize;
Modified:
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=672031&r1=672030&r2=672031&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
(original)
+++ lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
Thu Jun 26 14:14:06 2008
@@ -49,6 +49,8 @@
*
* @version $Id$
* @since solr 0.9
+ *
+ * @deprecated Use [EMAIL PROTECTED] DirectUpdateHandler2} instead. This is
only kept around for back-compatibility (way back).
*/
public class DirectUpdateHandler extends UpdateHandler {
@@ -236,7 +238,7 @@
closeSearcher(); // flush any deletes
if (cmd.optimize) {
openWriter(); // writer needs to be open to optimize
- writer.optimize();
+ writer.optimize(cmd.maxOptimizeSegments);
}
closeWriter();
Modified:
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=672031&r1=672030&r2=672031&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
(original)
+++ lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
Thu Jun 26 14:14:06 2008
@@ -360,8 +360,8 @@
if (cmd.optimize) {
closeSearcher();
- openWriter();
- writer.optimize();
+ openWriter();
+ writer.optimize(cmd.maxOptimizeSegments);
}
closeSearcher();
@@ -402,8 +402,6 @@
SolrException.log(log,e);
}
}
-
- return;
}
@@ -521,7 +519,8 @@
try {
CommitUpdateCommand command = new CommitUpdateCommand( false );
command.waitFlush = true;
- command.waitSearcher = true;
+ command.waitSearcher = true;
+ //no need for command.maxOptimizeSegments = 1; since it is not
optimizing
commit( command );
autoCommitCount++;
}
Added:
lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java?rev=672031&view=auto
==============================================================================
---
lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java
(added)
+++
lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java
Thu Jun 26 14:14:06 2008
@@ -0,0 +1,94 @@
+package org.apache.solr.update;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.util.AbstractSolrTestCase;
+
+import java.io.File;
+import java.io.FileFilter;
+
+
+/**
+ *
+ *
+ **/
+public class DirectUpdateHandlerOptimizeTest extends AbstractSolrTestCase {
+
+ public String getSchemaFile() {
+ return "schema.xml";
+ }
+
+ public String getSolrConfigFile() {
+ return "solrconfig-duh-optimize.xml";
+ }
+
+
+ public void testOptimize() throws Exception {
+ SolrCore core = h.getCore();
+
+ UpdateHandler updater = core.getUpdateHandler();
+ AddUpdateCommand cmd = new AddUpdateCommand();
+ cmd.overwriteCommitted = true;
+ cmd.overwritePending = true;
+ cmd.allowDups = false;
+ //add just under the merge factor, so no segments are merged
+ //the merge factor is 1000 and the maxBufferedDocs is 2, so there should
be 500 segments (498 segs each w/ 2 docs, and 1 segment with 1 doc)
+ for (int i = 0; i < 999; i++) {
+ // Add a valid document
+ cmd.doc = new Document();
+ cmd.doc.add(new Field("id", "id_" + i, Field.Store.YES,
Field.Index.UN_TOKENIZED));
+ cmd.doc.add(new Field("subject", "subject_" + i, Field.Store.NO,
Field.Index.TOKENIZED));
+ updater.addDoc(cmd);
+ }
+
+ CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false);
+ updater.commit(cmtCmd);
+
+ String indexDir = core.getIndexDir();
+ assertNumSegments(indexDir, 500);
+
+ //now do an optimize
+ cmtCmd = new CommitUpdateCommand(true);
+ cmtCmd.maxOptimizeSegments = 250;
+ updater.commit(cmtCmd);
+ assertNumSegments(indexDir, 250);
+
+ cmtCmd.maxOptimizeSegments = -1;
+ try {
+ updater.commit(cmtCmd);
+ assertTrue(false);
+ } catch (IllegalArgumentException e) {
+ }
+ cmtCmd.maxOptimizeSegments = 1;
+ updater.commit(cmtCmd);
+ assertNumSegments(indexDir, 1);
+ }
+
+ private void assertNumSegments(String indexDir, int numSegs) {
+ File file = new File(indexDir);
+ File[] segs = file.listFiles(new FileFilter() {
+ public boolean accept(File file) {
+ return file.getName().endsWith("tii");
+ }
+ });
+ assertTrue("Wrong number of segments: " + segs.length + " does not equal:
" + numSegs, segs.length == numSegs);
+ }
+
+}
Propchange:
lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerOptimizeTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied:
lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig-duh-optimize.xml
(from r670009, lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml)
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig-duh-optimize.xml?p2=lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig-duh-optimize.xml&p1=lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml&r1=670009&r2=672031&rev=672031&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml (original)
+++ lucene/solr/trunk/src/test/test-files/solr/conf/solrconfig-duh-optimize.xml
Thu Jun 26 14:14:06 2008
@@ -34,7 +34,9 @@
unless overridden. -->
<!-- Values here affect all index writers and act as a default unless
overridden. -->
<useCompoundFile>false</useCompoundFile>
- <mergeFactor>10</mergeFactor>
+ <!-- Set a high merge factor, so we get lots of segments -->
+ <mergeFactor>1000</mergeFactor>
+ <maxBufferedDocs>2</maxBufferedDocs>
<!-- If both ramBufferSizeMB and maxBufferedDocs is set, then Lucene will
flush based on whichever limit is hit first.
-->
<!--<maxBufferedDocs>1000</maxBufferedDocs>-->
@@ -44,7 +46,7 @@
If both ramBufferSizeMB and maxBufferedDocs is set, then Lucene will flush
based on whichever limit is hit first.
-->
- <ramBufferSizeMB>32</ramBufferSizeMB>
+
<maxMergeDocs>2147483647</maxMergeDocs>
<maxFieldLength>10000</maxFieldLength>
<writeLockTimeout>1000</writeLockTimeout>
@@ -86,8 +88,8 @@
<mainIndex>
<!-- lucene options specific to the main on-disk lucene index -->
<useCompoundFile>false</useCompoundFile>
- <mergeFactor>10</mergeFactor>
- <ramBufferSizeMB>32</ramBufferSizeMB>
+ <mergeFactor>1000</mergeFactor>
+ <maxBufferedDocs>2</maxBufferedDocs>
<maxMergeDocs>2147483647</maxMergeDocs>
<maxFieldLength>10000</maxFieldLength>
