Update of /var/cvs/contributions/lucene/src/org/mmbase/module/lucene
In directory james.mmbase.org:/tmp/cvs-serv17877/org/mmbase/module/lucene
Modified Files:
Indexer.java Lucene.java MMBaseEntry.java
MMBaseIndexDefinition.java NewSearcher.java Searcher.java
Log Message:
See also:
http://cvs.mmbase.org/viewcvs/contributions/lucene/src/org/mmbase/module/lucene
Index: Indexer.java
===================================================================
RCS file:
/var/cvs/contributions/lucene/src/org/mmbase/module/lucene/Indexer.java,v
retrieving revision 1.51
retrieving revision 1.52
diff -u -b -r1.51 -r1.52
--- Indexer.java 29 Jan 2008 09:52:36 -0000 1.51
+++ Indexer.java 1 Feb 2008 11:08:21 -0000 1.52
@@ -34,7 +34,7 @@
*
* @author Pierre van Rooden
* @author Michiel Meeuwissen
- * @version $Id: Indexer.java,v 1.51 2008/01/29 09:52:36 michiel Exp $
+ * @version $Id: Indexer.java,v 1.52 2008/02/01 11:08:21 michiel Exp $
**/
public class Indexer {
@@ -298,7 +298,7 @@
} finally {
if (writer != null) try { writer.close();} catch (IOException ioe)
{ log.error(ioe); }
}
- EventManager.getInstance().propagateEvent(NewSearcher.EVENT);
+ EventManager.getInstance().propagateEvent(new
NewSearcher.Event(getName()));
return updated;
}
@@ -428,14 +428,14 @@
writer.optimize();
Directory.copy(fullIndex, getDirectory(), true);
Date lastFullIndex = setLastFullIndex(startTime);
- log.service("Full index finished at " + lastFullIndex + ". Total
nr documents in index '" + getName() + "': " + writer.docCount());
+ log.info("Full index finished at " + lastFullIndex + ". Copied " +
fullIndex + " to " + getDirectory() + " Total nr documents in index '" +
getName() + "': " + writer.docCount());
} catch (Exception e) {
addError(e.getMessage());
log.error("Cannot run FullIndex: " + e.getMessage(), e);
} finally {
if (writer != null) { try { writer.close(); } catch (IOException
ioe) { log.error("Can't close index writer: " + ioe.getMessage()); } }
}
- EventManager.getInstance().propagateEvent(NewSearcher.EVENT);
+ EventManager.getInstance().propagateEvent(new
NewSearcher.Event(getName()));
}
/**
@@ -462,10 +462,15 @@
document = new Document();
document.add(new Field("indexId", indexId, Field.Store.YES,
Field.Index.UN_TOKENIZED));
indexed++;
+ if (indexed % 100 == 0) {
+ log.service("Indexed " + indexed + " documents");
+ } else if (log.isDebugEnabled()) {
+ log.debug("Indexed " + indexed + " documents");
+ }
}
index(entry, document);
if (Thread.currentThread().isInterrupted()) {
- log.debug("Interrupted");
+ log.info("Interrupted");
break;
}
lastIdentifier = newIdentifier;
@@ -476,7 +481,7 @@
}
writer.addDocument(document);
}
- EventManager.getInstance().propagateEvent(NewSearcher.EVENT);
+ EventManager.getInstance().propagateEvent(new
NewSearcher.Event(getName()));
return indexed;
}
Index: Lucene.java
===================================================================
RCS file:
/var/cvs/contributions/lucene/src/org/mmbase/module/lucene/Lucene.java,v
retrieving revision 1.104
retrieving revision 1.105
diff -u -b -r1.104 -r1.105
--- Lucene.java 14 Jan 2008 18:33:35 -0000 1.104
+++ Lucene.java 1 Feb 2008 11:08:21 -0000 1.105
@@ -35,6 +35,7 @@
import java.util.concurrent.*;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.search.BooleanQuery;
@@ -47,7 +48,7 @@
*
* @author Pierre van Rooden
* @author Michiel Meeuwissen
- * @version $Id: Lucene.java,v 1.104 2008/01/14 18:33:35 michiel Exp $
+ * @version $Id: Lucene.java,v 1.105 2008/02/01 11:08:21 michiel Exp $
**/
public class Lucene extends ReloadableModule implements NodeEventListener,
RelationEventListener, IdEventListener {
@@ -466,6 +467,33 @@
}
+ protected Function/*<Void>*/ clearDirectory = new
AbstractFunction/*<Void>*/("clearDirectory", new Parameter[] {INDEX, COPY},
ReturnType.VOID) {
+ public Object getFunctionValue(Parameters arguments) {
+ String index = (String) arguments.getString(INDEX);
+ Indexer indexer = indexerMap.get(index);
+ boolean copy = Boolean.TRUE.equals(arguments.get(COPY));
+ try {
+ Directory dir = copy ? indexer.getDirectoryForFullIndex():
indexer.getDirectory();
+ for (String file : dir.list()) {
+ if (file != null) {
+ try {
+ dir.deleteFile(file);
+ } catch (Exception e) {
+ log.warn(e);
+ }
+ }
+ }
+ if (! copy) EventManager.getInstance().propagateEvent(new
NewSearcher.Event(index));
+ } catch (java.io.IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ return null;
+ }
+ };
+ {
+ addFunction(clearDirectory);
+ }
+
//protected Function<Integer> unAssignFunction = new
AbstractFunction<Integer>("unassign", new Parameter("id", Integer.class, true))
{
protected Function/*<Integer>*/ unAssignFunction = new
AbstractFunction/*<Integer>*/("unassign", new Parameter[] {new Parameter("id",
Integer.class, true)}, ReturnType.INTEGER) {
public Integer getFunctionValue(Parameters arguments) {
@@ -502,6 +530,8 @@
addFunction(interruptFunction);
}
+
+
//protected Function<Date> lastFullIndexFunction = new
AbstractFunction<Date>("last", INDEX) {
protected Function/*<Date>*/ lastFullIndexFunction = new
AbstractFunction/*<Date>*/("last", new Parameter[] {INDEX}, ReturnType.UNKNOWN)
{
public Date getFunctionValue(Parameters arguments) {
Index: MMBaseEntry.java
===================================================================
RCS file:
/var/cvs/contributions/lucene/src/org/mmbase/module/lucene/MMBaseEntry.java,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -b -r1.29 -r1.30
--- MMBaseEntry.java 27 Dec 2007 13:25:00 -0000 1.29
+++ MMBaseEntry.java 1 Feb 2008 11:08:21 -0000 1.30
@@ -34,7 +34,7 @@
*
* @author Pierre van Rooden
* @author Michiel Meeuwissen
- * @version $Id: MMBaseEntry.java,v 1.29 2007/12/27 13:25:00 pierre Exp $
+ * @version $Id: MMBaseEntry.java,v 1.30 2008/02/01 11:08:21 michiel Exp $
**/
public class MMBaseEntry implements IndexEntry {
static private final Logger log =
Logging.getLoggerInstance(MMBaseEntry.class);
@@ -126,12 +126,13 @@
String value = getFieldDataAsString(data, fieldName);
if (fieldDefinition.escaper != null) {
org.mmbase.util.transformers.CharTransformer transformer =
null;
- try {
- transformer =
org.mmbase.bridge.jsp.taglib.ContentTag.getCharTransformer(fieldDefinition.escaper,null);
- } catch (javax.servlet.jsp.JspTagException jte) {
+ //try {
+ transformer = null;
+
//org.mmbase.bridge.jsp.taglib.ContentTag.getCharTransformer(fieldDefinition.escaper,
null);
+ //} catch (javax.servlet.jsp.JspTagException jte) {
// ignore if an escaper does not exist for now
(otherwise log fills up)
- }
- if (transformer !=null) {
+ //}
+ if (transformer != null) {
value = transformer.transform(value);
}
}
Index: MMBaseIndexDefinition.java
===================================================================
RCS file:
/var/cvs/contributions/lucene/src/org/mmbase/module/lucene/MMBaseIndexDefinition.java,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -b -r1.22 -r1.23
--- MMBaseIndexDefinition.java 18 Dec 2007 12:17:11 -0000 1.22
+++ MMBaseIndexDefinition.java 1 Feb 2008 11:08:21 -0000 1.23
@@ -26,7 +26,7 @@
* fields can have extra attributes specific to Lucene searching.
*
* @author Pierre van Rooden
- * @version $Id: MMBaseIndexDefinition.java,v 1.22 2007/12/18 12:17:11 pierre
Exp $
+ * @version $Id: MMBaseIndexDefinition.java,v 1.23 2008/02/01 11:08:21 michiel
Exp $
**/
class MMBaseIndexDefinition extends QueryDefinition implements IndexDefinition
{
static private final Logger log =
Logging.getLoggerInstance(MMBaseIndexDefinition.class);
@@ -55,7 +55,12 @@
private final Map<String, Float> boosts = new HashMap<String, Float>();
- private final BasicReleaseStrategy releaseStrategy = new
BasicReleaseStrategy();
+ private final ChainedReleaseStrategy releaseStrategy = new
ChainedReleaseStrategy();
+ {
+ releaseStrategy.addReleaseStrategy(new BetterStrategy());
+ releaseStrategy.addReleaseStrategy(new ConstraintsMatchingStrategy());
+ }
+
MMBaseIndexDefinition() {
}
@@ -98,9 +103,16 @@
public boolean inIndex(String identifier) {
Cloud cloud = query.getCloud();
- if (! cloud.hasNode(identifier)) return false;
+ if (! cloud.hasNode(identifier)) {
+ log.debug("No such node " + identifier + " (" + getId() + ")");
+ return false;
+ }
Node node = cloud.getNode(identifier);
- return
releaseStrategy.evaluate(NodeEventHelper.createNodeEventInstance(node,
Event.TYPE_NEW, null), query, null).shouldRelease();
+ NodeEvent pseudoEvent = new NodeEvent(null,
node.getNodeManager().getName(), node.getNumber(), Collections.EMPTY_MAP,
+ new NodeMap(node),
Event.TYPE_CHANGE);
+ boolean result = releaseStrategy.evaluate(pseudoEvent, query,
null).shouldRelease();
+ log.debug("Node " + identifier + (result ? " IS " : " IS NOT ") + "in
index " + getId() + " according to " + releaseStrategy);
+ return result;
}
/**
* Converts an MMBase Node Iterator to an Iterator of IndexEntry-s.
@@ -177,8 +189,8 @@
}
StepField elementNumberField =
q.createStepField(elementNumberFieldName);
q.addSortOrder(elementNumberField, SortOrder.ORDER_DESCENDING); //
this sort order makes it possible to filter out duplicates.
- if (log.isDebugEnabled()) {
- log.debug("Query for node '" + id + "': " + q.toSql());
+ if (log.isTraceEnabled()) {
+ log.trace("Query for node '" + id + "': " + q.toSql());
}
return new HugeNodeListIterator(q, maxNodesInQuery);
} catch (Exception e) {
Index: NewSearcher.java
===================================================================
RCS file:
/var/cvs/contributions/lucene/src/org/mmbase/module/lucene/NewSearcher.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -b -r1.3 -r1.4
--- NewSearcher.java 17 Jan 2007 22:00:14 -0000 1.3
+++ NewSearcher.java 1 Feb 2008 11:08:21 -0000 1.4
@@ -15,23 +15,25 @@
* All the code necessary to propagate events with the meaning 'the lucene
index has changed, you must refresh your searchers'.
* @since MMBase-1.9
* @author Michiel Meeuwissen
- * @version $Id: NewSearcher.java,v 1.3 2007/01/17 22:00:14 michiel Exp $
+ * @version $Id: NewSearcher.java,v 1.4 2008/02/01 11:08:21 michiel Exp $
*/
public class NewSearcher {
- /**
- * The event itself
- */
public static class Event extends org.mmbase.core.event.Event {
- private Event() {};
- }
+ protected final String index;
/**
- * No further structure, so it can as well be a singleton
+ * The event itself
*/
- public static Event EVENT = new Event();
+ public Event(String i) {
+ index = i;
+ };
+ public String getIndex() {
+ return index;
+ }
+ }
/**
* The associated listener
Index: Searcher.java
===================================================================
RCS file:
/var/cvs/contributions/lucene/src/org/mmbase/module/lucene/Searcher.java,v
retrieving revision 1.43
retrieving revision 1.44
diff -u -b -r1.43 -r1.44
--- Searcher.java 14 Jan 2008 17:49:48 -0000 1.43
+++ Searcher.java 1 Feb 2008 11:08:21 -0000 1.44
@@ -33,7 +33,7 @@
* A wrapper around Lucene's [EMAIL PROTECTED]
org.apache.lucene.search.IndexSearcher}. Every [EMAIL PROTECTED] Indexer} has
its own Searcher.
*
* @author Pierre van Rooden
- * @version $Id: Searcher.java,v 1.43 2008/01/14 17:49:48 michiel Exp $
+ * @version $Id: Searcher.java,v 1.44 2008/02/01 11:08:21 michiel Exp $
* @todo Should the StopAnalyzers be replaced by index.analyzer? Something
else?
**/
public class Searcher implements NewSearcher.Listener {
@@ -62,24 +62,23 @@
this.index = index;
searchLog = Logging.getLoggerInstance("org.mmbase.lucene.SEARCH." +
index.getName());
this.allIndexedFields = allIndexedFields;
- try {
- searcher = new IndexSearcher(index.getDirectory());
- } catch (IOException ioe) {
- log.error("Can't close index searcher: " + ioe.getMessage());
- }
EventManager.getInstance().addEventListener(this);
}
public void notify(NewSearcher.Event event) {
+ if (event.getIndex().equals(index.getName())) {
+ log.debug("Received " + event);
needsNewSearcher = true;
}
+ }
- protected IndexSearcher getSearcher(boolean copy) {
- if (needsNewSearcher || searcher == null) {
+ protected synchronized IndexSearcher getSearcher(boolean copy) throws
IOException {
+ if (copy) return new IndexSearcher(index.getDirectoryForFullIndex() );
+ if (searcher != null && needsNewSearcher) {
// for existing searches, leave existing searcher open for 10
seconds, then close it (searches still not finished in 10 seconds, get an IO
exception)
- if (searcher != null) {
closingSearchers++;
final IndexSearcher s = searcher;
+ searcher = null;
timer.schedule(new TimerTask() {
public void run() {
try {
@@ -93,12 +92,9 @@
}
}, 10000);
}
- try {
+ if (searcher == null) {
+ searcher = new IndexSearcher(index.getDirectory());
needsNewSearcher = false;
- searcher = new IndexSearcher(copy ?
index.getDirectoryForFullIndex() : index.getDirectory());
- } catch (IOException ioe) {
- log.error("Can't close index searcher: " + ioe.getMessage());
- }
return searcher;
} else {
return searcher;
@@ -111,6 +107,7 @@
try {
log.service("Shutting down searcher for " + index);
searcher.close();
+ searcher = null;
} catch (IOException ioe) {
log.error("Can't close index searcher: " + ioe.getMessage());
}
@@ -201,7 +198,8 @@
try {
hits = getHits(value, filter, sort, analyzer, extraQuery,
fields, false);
} catch (java.io.IOException ioe) {
- log.service(ioe + " returning empty list");
+ log.service(ioe + " returning empty list", ioe);
+ needsNewSearcher = true;
return org.mmbase.bridge.util.BridgeCollections.EMPTY_NODELIST;
}
if (log.isTraceEnabled()) {
@@ -303,9 +301,9 @@
booleanQuery.add(extraQuery, BooleanClause.Occur.MUST);
query = booleanQuery;
}
- IndexSearcher searcher = getSearcher(copy);
- if (searcher == null) throw new IOException("No IndexSearcher found
for " + this);
- return searcher.search(query, filter, sort);
+ IndexSearcher s = getSearcher(copy);
+ if (s == null) throw new IOException("No IndexSearcher found for " +
this);
+ return s.search(query, filter, sort);
}
_______________________________________________
Cvs mailing list
[email protected]
http://lists.mmbase.org/mailman/listinfo/cvs
