Update of /var/cvs/contributions/lucene/src/org/mmbase/module/lucene
In directory james.mmbase.org:/tmp/cvs-serv6984
Modified Files:
FullIndexEvents.java Lucene.java NewSearcher.java
Added Files:
AssignmentEvents.java
Log Message:
LUCENE-13, plus some generic cleaning
See also:
http://cvs.mmbase.org/viewcvs/contributions/lucene/src/org/mmbase/module/lucene
See also: http://www.mmbase.org/jira/browse/LUCENE-13
AssignmentEvents.java is new
Index: FullIndexEvents.java
===================================================================
RCS file:
/var/cvs/contributions/lucene/src/org/mmbase/module/lucene/FullIndexEvents.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -b -r1.1 -r1.2
--- FullIndexEvents.java 1 Feb 2008 12:40:42 -0000 1.1
+++ FullIndexEvents.java 28 Jul 2008 13:03:36 -0000 1.2
@@ -12,10 +12,11 @@
import org.mmbase.core.event.*;
/**
- * All the code necessary to propagate events with the meaning 'the lucene
index has changed, you must refresh your searchers'.
+ * Events send during the execution of a full index.
+ *
* @since MMBase-1.9
* @author Michiel Meeuwissen
- * @version $Id: FullIndexEvents.java,v 1.1 2008/02/01 12:40:42 michiel Exp $
+ * @version $Id: FullIndexEvents.java,v 1.2 2008/07/28 13:03:36 michiel Exp $
*/
public class FullIndexEvents {
@@ -74,9 +75,10 @@
Listener nel = (Listener) listener;
nel.notify(ne);
}
+ public String toString() {
+ return "Lucene Full Index Broker";
+ }
}
-
-
}
Index: Lucene.java
===================================================================
RCS file:
/var/cvs/contributions/lucene/src/org/mmbase/module/lucene/Lucene.java,v
retrieving revision 1.112
retrieving revision 1.113
diff -u -b -r1.112 -r1.113
--- Lucene.java 21 Jul 2008 14:26:34 -0000 1.112
+++ Lucene.java 28 Jul 2008 13:03:36 -0000 1.113
@@ -48,9 +48,9 @@
*
* @author Pierre van Rooden
* @author Michiel Meeuwissen
- * @version $Id: Lucene.java,v 1.112 2008/07/21 14:26:34 michiel Exp $
+ * @version $Id: Lucene.java,v 1.113 2008/07/28 13:03:36 michiel Exp $
**/
-public class Lucene extends ReloadableModule implements NodeEventListener,
RelationEventListener, IdEventListener {
+public class Lucene extends ReloadableModule implements NodeEventListener,
RelationEventListener, IdEventListener, AssignmentEvents.Listener {
public static final String PUBLIC_ID_LUCENE_2_0 = "-//MMBase//DTD
luceneindex config 2.0//EN";
public static final String DTD_LUCENE_2_0 = "luceneindex_2_0.dtd";
@@ -81,6 +81,9 @@
protected final static Parameter/*<String>*/ INDEX = new
Parameter("index", String.class);
static { INDEX.setDescription("the name of the index to search in"); }
+ protected final static Parameter/*<List>*/ MACHINES = new
Parameter("machines", List.class);
+ static { MACHINES.setDescription("the machines on which to execute a full
index"); }
+
protected final static Parameter/*<Boolean>*/ COPY = new Parameter("copy",
Boolean.class, Boolean.FALSE);
static { INDEX.setDescription("use the copy of the index (used for full
index)"); }
@@ -123,11 +126,8 @@
XMLEntityResolver.registerPublicID(PUBLIC_ID_LUCENE, DTD_LUCENE,
Lucene.class);
}
- // initial wait time after startup, default 2 minutes
- private static final long INITIAL_WAIT_TIME = 2 * 60 * 1000;
- // wait time bewteen individual checks, default 5 seconds
private static final long WAIT_TIME = 5 * 1000;
- // default path to the lucene data
+
private static final String INDEX_CONFIG_FILE = "utils/luceneindex.xml";
private static final Logger log = Logging.getLoggerInstance(Lucene.class);
@@ -137,13 +137,7 @@
return (Lucene) getModule("lucene");
}
- /**
- * The MMBase instance, used for low-level access
- */
- protected MMBase mmbase = null;
- private long initialWaitTime = INITIAL_WAIT_TIME;
- private long waitTime = WAIT_TIME;
private String indexPath = null;
private Scheduler scheduler = null;
private String defaultIndex = null;
@@ -151,6 +145,10 @@
private final Map<String, Indexer> indexerMap = new
ConcurrentHashMap<String, Indexer>();
private final Map<String, Searcher> searcherMap = new
ConcurrentHashMap<String, Searcher>();
private boolean readOnly = false;
+ private String master; // If readonly, the machine name of the mmbase
which is responsible for the index
+
+ private long waitTime = WAIT_TIME;
+
private List<String> configErrors = new ArrayList<String>();
private Date configReadTime = new Date(0);
@@ -188,20 +186,31 @@
* This function can be called through the function framework.
*/
//protected Function<Void> fullIndexFunction = new
AbstractFunction<Void>("fullIndex", INDEX) {
- protected Function/*<Void>*/ fullIndexFunction = new
AbstractFunction/*<Void>*/("fullIndex", new Parameter[] {INDEX},
ReturnType.VOID) {
+ protected Function/*<Void>*/ fullIndexFunction = new
AbstractFunction/*<Void>*/("fullIndex", new Parameter[] {INDEX, MACHINES},
ReturnType.VOID) {
public Object getFunctionValue(Parameters arguments) {
- if (scheduler == null) throw new RuntimeException("Read only");
String index = (String) arguments.get(INDEX);
+ List<String> machines = (List<String>) arguments.get(MACHINES);
+ EventManager.getInstance().propagateEvent(new
AssignmentEvents.Event(index, machines));
+ return null;
+ }
+ };
+ {
+ addFunction(fullIndexFunction);
+ }
+
+ public void notify(AssignmentEvents.Event event) {
+ log.info("Received " + event);
+ if (event.getMachines().contains(MMBase.getMMBase().getMachineName()))
{
+ String index = event.getIndex();
+ if (scheduler == null) throw new RuntimeException("Read only");
if (index == null || "".equals(index)) {
scheduler.fullIndex();
} else {
scheduler.fullIndex(index);
}
- return null;
+ } else {
+ log.info("Event " + event + " ignored");
}
- };
- {
- addFunction(fullIndexFunction);
}
/**
@@ -210,7 +219,7 @@
* the right index is addressed.
*/
//protected Function<Void> deleteIndexFunction = new
AbstractFunction<Void>("deleteIndex", INDEX, IDENTIFIER, CLASS) {
- protected Function/*<Void>*/ deleteIndexFunction = new
AbstractFunction/*<Void>*/("deleteIndex", new Parameter[] {INDEX, IDENTIFIER,
CLASS}, ReturnType.VOID) {
+ protected Function/*<Void>*/ deleteIndexFunction = new
AbstractFunction/*<Void>*/("deleteIndex", new Parameter[] {INDEX, MACHINES,
IDENTIFIER, CLASS}, ReturnType.VOID) {
public Object getFunctionValue(Parameters arguments) {
if (scheduler == null) throw new RuntimeException("Read only");
if(!readOnly){
@@ -319,13 +328,19 @@
//protected Function<Boolean> readOnlyFunction = new
AbstractFunction<Boolean>("readOnly"){
- protected Function /*<Boolean>*/ readOnlyFunction = new AbstractFunction
/*<Boolean>*/("readOnly", Parameter.EMPTY, ReturnType.BOOLEAN){
+ protected Function /*<Boolean>*/ readOnlyFunction = new AbstractFunction
/*<Boolean>*/("readOnly", Parameter.EMPTY, ReturnType.BOOLEAN) {
public Boolean getFunctionValue(Parameters arguments) {
return readOnly;
}
};
{
addFunction(readOnlyFunction);
+
+ addFunction(new AbstractFunction/*<String>*/("master",
Parameter.EMPTY, ReturnType.STRING) {
+ public String getFunctionValue(Parameters argumnets) {
+ return master;
+ }
+ });
}
/**
@@ -475,7 +490,7 @@
}
- protected Function/*<Void>*/ clearDirectory = new
AbstractFunction/*<Void>*/("clearDirectory", new Parameter[] {INDEX, COPY},
ReturnType.VOID) {
+ protected Function/*<Void>*/ clearDirectory = new
AbstractFunction/*<Void>*/("clearDirectory", new Parameter[] {INDEX, MACHINES,
COPY}, ReturnType.VOID) {
public Object getFunctionValue(Parameters arguments) {
if (readOnly) {
throw new IllegalStateException("This lucene is readonly");
@@ -628,28 +643,7 @@
ThreadPools.jobsExecutor.execute(new Runnable() {
public void run() {
// Force init of MMBase
- mmbase = MMBase.getMMBase();
-
- if (initialWait) {
- // initial wait time?
- String time = getInitParameter("initialwaittime");
- if (time != null) {
- try {
- initialWaitTime = Long.parseLong(time);
- log.debug("Set initial wait time to " + time +
" milliseconds");
- } catch (NumberFormatException nfe) {
- log.warn("Invalid value '" + time + "' for
property 'initialwaittime'");
- }
- }
- try {
- if (initialWaitTime > 0) {
- log.info("Sleeping " + (initialWaitTime /
1000) + " seconds for initialisation");
- Thread.sleep(initialWaitTime);
- }
- } catch (InterruptedException ie) {
- //return;
- }
- }
+ MMBase mmbase = MMBase.getMMBase();
factory = ContentExtractor.getInstance();
@@ -691,36 +685,36 @@
try {
boolean write =
java.net.InetAddress.getLocalHost().getHostName().equals(host) ||
- (System.getProperty("catalina.base") + "@"
+ java.net.InetAddress.getLocalHost().getHostName()).equals(host);
+ (System.getProperty("catalina.base") + "@"
+ java.net.InetAddress.getLocalHost().getHostName()).equals(host) ||
+ mmbase.getMachineName().equals(host);
readOnly = ! write;
+ if (readOnly) {
+ master = getInitParameter("master");
+ if (master == null) {
+ master = host;
+ }
+ }
} catch (java.net.UnknownHostException uhe) {
log.error(uhe);
}
} else {
readOnly = "true".equals(readOnlySetting);
+ if (readOnly) {
+ master = getInitParameter("master");
+ }
}
}
if (readOnly) {
- log.info("Lucene module of this MMBase will be
READONLY");
+ log.info("Lucene module of this MMBase will be
READONLY. Responsible for the index is " + (master != null ? master :
"UKNOWN"));
}
String time = getInitParameter("waittime");
if (time != null) {
try {
waitTime = Long.parseLong(time);
- log.debug("Set wait time to " + time + "
milliseconds");
+ log.debug("Set wait time to " + time + "
milliseconds. This long assigments remain scheduled, and can still be
canceled.");
} catch (NumberFormatException nfe) {
- log.warn("Invalid value '" + time +" ' for
property 'iwaittime'");
- }
- }
- while(! mmbase.getState()) {
- if (mmbase.isShutdown()) break;
- try {
- log.service("MMBase not yet up, waiting for 10
seconds.");
- Thread.sleep(10000);
- } catch (InterruptedException ie) {
- log.info(ie);
- return;
+ log.warn("Invalid value '" + time +" ' for
property 'waittime'");
}
}
@@ -738,7 +732,7 @@
log.service("Module Lucene started");
// full index ?
String fias = getInitParameter("fullindexatstartup");
- if (initialWaitTime < 0 || "true".equals(fias)) {
+ if ("true".equals(fias)) {
scheduler.fullIndex();
}
} else {
@@ -776,6 +770,8 @@
public void shutdown() {
+ EventManager.getInstance().removeEventListener(assignmentsListener);
+ EventManager.getInstance().removeEventListener(idListener);
if (scheduler != null) {
log.service("Stopping Lucene Scheduler");
scheduler.interrupt();
@@ -786,7 +782,7 @@
}
searcherMap.clear();
scheduler = null;
- mmbase = null;
+
}
public void reload() {
@@ -830,6 +826,7 @@
// MM: I think the follwing functionality should be present on
MMBaseIndexDefinition itself. and not on Lucene.
// And of course, the new event-mechanism must be used.
if (!readOnly) {
+ MMBase mmbase = MMBase.getMMBase();
// register. Unfortunately this can currently only be done
through the core
//for (Step step : queryDefinition.query.getSteps() ) {
for (Iterator i = queryDefinition.query.getSteps().iterator();
i.hasNext();) {
@@ -874,6 +871,16 @@
return Lucene.this.toString();
}
};
+ protected final AssignmentEvents.Listener assignmentsListener = new
AssignmentEvents.Listener() {
+ // wrapping to avoid also registring it as a NodeEventListener
+ public void notify(AssignmentEvents.Event event) {
+ Lucene.this.notify(event);
+ }
+ public String toString() {
+ return Lucene.this.toString();
+ }
+ };
+
protected void readConfiguration(String resource) {
@@ -983,6 +990,8 @@
}
}
} else if ("jdbc".equals(name)) {
+ MMBase mmbase = MMBase.getMMBase();
+
DataSource ds =
((DatabaseStorageManagerFactory)
mmbase.getStorageManagerFactory()).getDataSource();
IndexDefinition id = new
JdbcIndexDefinition(ds, childElement,
allIndexedFieldsSet, storeText, mergeText, analyzer, false);
@@ -1003,6 +1012,7 @@
}
}
}
+
Indexer indexer = new Indexer(indexPath, indexName,
queries, analyzer, readOnly);
for (String s : configErrors) {
indexer.addError(url.toString() + ": " + s);
@@ -1020,6 +1030,7 @@
log.warn("Can't read Lucene configuration: "+ e.getMessage(),
e);
}
}
+ EventManager.getInstance().addEventListener(assignmentsListener);
configReadTime = new Date();
}
@@ -1127,6 +1138,7 @@
}
public void run() {
+ MMBase mmbase = MMBase.getMMBase();
log.service("Start Lucene Scheduler");
while (mmbase != null && !mmbase.isShutdown()) {
if (log.isTraceEnabled()) {
Index: NewSearcher.java
===================================================================
RCS file:
/var/cvs/contributions/lucene/src/org/mmbase/module/lucene/NewSearcher.java,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -b -r1.5 -r1.6
--- NewSearcher.java 21 Jul 2008 14:30:18 -0000 1.5
+++ NewSearcher.java 28 Jul 2008 13:03:36 -0000 1.6
@@ -13,9 +13,8 @@
/**
* All the code necessary to propagate events with the meaning 'the lucene
index has changed, you must refresh your searchers'.
- * @since MMBase-1.9
* @author Michiel Meeuwissen
- * @version $Id: NewSearcher.java,v 1.5 2008/07/21 14:30:18 michiel Exp $
+ * @version $Id: NewSearcher.java,v 1.6 2008/07/28 13:03:36 michiel Exp $
*/
public class NewSearcher {
@@ -61,6 +60,9 @@
Listener nel = (Listener) listener;
nel.notify(ne);
}
+ public String toString() {
+ return "Lucene New Searcher Broker";
+ }
}
_______________________________________________
Cvs mailing list
[email protected]
http://lists.mmbase.org/mailman/listinfo/cvs
