knguyen 2005/06/08 22:53:46 CEST
Modified files: (Branch: JAHIA-4-1-BRANCH)
src/java/org/jahia/services/search AddedField.java
JahiaSearchBaseService.java
RAMIndexer.java
RemovedField.java
ScheduledSiteIndexationJob.java
SearchEngine.java
SiteIndexationStatus.java
Log:
search
Revision Changes Path
1.14.2.8.2.8 +2 -1
jahia/src/java/org/jahia/services/search/AddedField.java
http://jahia.mine.nu:8080/cgi-bin/cvsweb.cgi/jahia/src/java/org/jahia/services/search/AddedField.java.diff?r1=1.14.2.8.2.7&r2=1.14.2.8.2.8&f=h
1.42.2.15.2.20 +97 -30
jahia/src/java/org/jahia/services/search/JahiaSearchBaseService.java
http://jahia.mine.nu:8080/cgi-bin/cvsweb.cgi/jahia/src/java/org/jahia/services/search/JahiaSearchBaseService.java.diff?r1=1.42.2.15.2.19&r2=1.42.2.15.2.20&f=h
1.1.2.4 +64 -25
jahia/src/java/org/jahia/services/search/RAMIndexer.java
http://jahia.mine.nu:8080/cgi-bin/cvsweb.cgi/jahia/src/java/org/jahia/services/search/RAMIndexer.java.diff?r1=1.1.2.3&r2=1.1.2.4&f=h
1.3.4.2.2.1 +4 -2
jahia/src/java/org/jahia/services/search/RemovedField.java
http://jahia.mine.nu:8080/cgi-bin/cvsweb.cgi/jahia/src/java/org/jahia/services/search/RemovedField.java.diff?r1=1.3.4.2&r2=1.3.4.2.2.1&f=h
1.1.2.8 +1 -0
jahia/src/java/org/jahia/services/search/ScheduledSiteIndexationJob.java
http://jahia.mine.nu:8080/cgi-bin/cvsweb.cgi/jahia/src/java/org/jahia/services/search/ScheduledSiteIndexationJob.java.diff?r1=1.1.2.7&r2=1.1.2.8&f=h
1.17.2.1.2.5 +4 -1
jahia/src/java/org/jahia/services/search/SearchEngine.java
http://jahia.mine.nu:8080/cgi-bin/cvsweb.cgi/jahia/src/java/org/jahia/services/search/SearchEngine.java.diff?r1=1.17.2.1.2.4&r2=1.17.2.1.2.5&f=h
1.1.2.2 +4 -4
jahia/src/java/org/jahia/services/search/SiteIndexationStatus.java
http://jahia.mine.nu:8080/cgi-bin/cvsweb.cgi/jahia/src/java/org/jahia/services/search/SiteIndexationStatus.java.diff?r1=1.1.2.1&r2=1.1.2.2&f=h
Index: AddedField.java
===================================================================
RCS file:
/cvs/jahia/src/java/org/jahia/services/search/Attic/AddedField.java,v
retrieving revision 1.14.2.8.2.7
retrieving revision 1.14.2.8.2.8
diff -u -r1.14.2.8.2.7 -r1.14.2.8.2.8
--- AddedField.java 24 May 2005 18:13:40 -0000 1.14.2.8.2.7
+++ AddedField.java 8 Jun 2005 20:53:45 -0000 1.14.2.8.2.8
@@ -48,10 +48,11 @@
* @param key
* @param fields
*/
+ /*
public AddedField (int siteId, String key,
Hashtable fields) {
super (siteId, null, key, fields);
- }
+ }*/
/**
* A Field instance to store in the search index
Index: JahiaSearchBaseService.java
===================================================================
RCS file:
/cvs/jahia/src/java/org/jahia/services/search/Attic/JahiaSearchBaseService.java,v
retrieving revision 1.42.2.15.2.19
retrieving revision 1.42.2.15.2.20
diff -u -r1.42.2.15.2.19 -r1.42.2.15.2.20
--- JahiaSearchBaseService.java 3 Jun 2005 16:54:46 -0000
1.42.2.15.2.19
+++ JahiaSearchBaseService.java 8 Jun 2005 20:53:45 -0000
1.42.2.15.2.20
@@ -271,7 +271,7 @@
System.setProperty("org.apache.lucene.minMergeDocs",
String.valueOf(1000));
}
- val =
searchSettings.getProperty("org.jahia.services.search.scheduledJobInterval");
+ val = searchSettings.getProperty("scheduledJobInterval");
if ( val != null ){
try {
this.scheduledIndexationInterval = Long.parseLong(val);
@@ -1006,7 +1006,8 @@
boolean result = false;
try {
- String indexDirectory = this.getSiteIndex(siteID);
+ JahiaSite site =
ServicesRegistry.getInstance().getJahiaSitesService().getSite(siteID);
+ String indexDirectory = composeSiteIndexDir (site);
if ( !IndexReader.indexExists(indexDirectory) ){
writer = getIndexWriter (siteID, this.indexAnalyzer, true);
} else {
@@ -1077,19 +1078,60 @@
if ( this.isInitialized() && this.localIndexing ){
this.optimizeAllIndexes();
}
-
+
SiteIndexer siteIndexer = null;
Integer siteId = null;
HashMap siteIndexers = new HashMap();
long lastOptimizationTime = System.currentTimeMillis();
+ Vector v = null;
+ Vector validDocs = null;
+ JahiaIndexableDocument doc = null;
+ JahiaIndexableDocument doc2 = null;
+ Iterator iterator = null;
+ RemovedField remField = null;
+ AddedField addField = null;
+ long now = 0;
while (this.isInitialized() && indexingThreadActivated) {
- Vector v = new Vector();
+ now = System.currentTimeMillis();
+ v = new Vector();
+ doc = null;
- HashMap toBeAdded = new HashMap();
- HashMap toBeRemoved = new HashMap();
- JahiaIndexableDocument doc = null;
synchronized (this) {
+ for ( int i=0; i<indexOrders.size(); i++ ){
+ doc = (JahiaIndexableDocument)indexOrders.get(i);
+ if (
doc.scheduled((IndexationJobDetail)this.scheduledIndexationJob) ){
+ continue;
+ }
+ if ( doc instanceof RemovedField ) {
+ remField = (RemovedField)doc;
+ for ( int j=0; j<v.size() ; j++ ){
+ doc2 = (JahiaIndexableDocument)v.get(j);
+ if ( doc2 != null && doc2 instanceof AddedField
){
+ addField = (AddedField)doc2;
+ if ( remField.getID() == addField.getID() ){
+ // we are going to remove the same
field, so we don't need to add it at all.
+ v.setElementAt(null,j);
+ }
+ }
+ }
+ // process remove first
+ v.insertElementAt(doc,0);
+ } else {
+ // process add at last
+ v.add(doc);
+ }
+ }
+ validDocs = new Vector();
+ for ( int i=0; i<v.size() ; i++ ){
+ doc = (JahiaIndexableDocument)v.get(i);
+ if ( doc != null){
+ validDocs.add(doc);
+ }
+ }
+ v = null;
+
+ /*
// 1.Separate docs that are going to be added or removed
for ( int i=0; i<indexOrders.size(); i++ ){
doc = (JahiaIndexableDocument)indexOrders.get(i);
@@ -1105,10 +1147,10 @@
}
// 2. remove "to added" docs, if they are going to be removed
- Iterator iterator = toBeRemoved.keySet().iterator();
- String key = null;
- Integer addOrder = null;
- Integer removeOrder = null;
+ iterator = toBeRemoved.keySet().iterator();
+ key = null;
+ addOrder = null;
+ removeOrder = null;
while ( iterator.hasNext() ){
key = (String)iterator.next();
addOrder = (Integer)toBeAdded.get(key);
@@ -1121,30 +1163,31 @@
}
}
}
+ */
indexOrders = new Vector();
}
long indexingStartTime = System.currentTimeMillis();
- int indexOrderCount = v.size ();
+ int indexOrderCount = validDocs.size ();
- while (v.size() > 0) {
- JahiaIndexableDocument nextObject = null;
- if (v.size() != 0) {
- nextObject = (JahiaIndexableDocument)
- v.elementAt(0);
- v.remove(0);
+ while (validDocs.size() > 0) {
+ doc = null;
+ if (validDocs.size() != 0) {
+ doc = (JahiaIndexableDocument)
+ validDocs.elementAt(0);
+ validDocs.remove(0);
}
// okay now we have the next added/removed field, we process
it!
- if (nextObject != null) {
+ if (doc != null) {
- siteId = new Integer(nextObject.getSiteId());
+ siteId = new Integer(doc.getSiteId());
siteIndexer = (SiteIndexer)siteIndexers.get(siteId);
try {
if ( siteIndexer == null ){
- siteIndexer = new
SiteIndexer(siteId.intValue(),1000);
+ siteIndexer = new
SiteIndexer(siteId.intValue(),500);
siteIndexers.put(siteId,siteIndexer);
}
} catch ( Throwable t ){
@@ -1152,18 +1195,17 @@
}
if ( siteIndexer != null ) {
try {
- siteIndexer.addDocument(nextObject);
+ siteIndexer.addDocument(doc);
} catch ( Throwable t ){
logger.debug("Error addind document to
SiteIndexer siteId="
+ siteId.intValue(), t);
}
}
- this.indexOrdersCache.remove(getCacheKey(nextObject));
+ this.indexOrdersCache.remove(getCacheKey(doc));
}
}
- long now = System.currentTimeMillis();
- Iterator iterator = siteIndexers.values().iterator();
+ iterator = siteIndexers.values().iterator();
while (iterator.hasNext()) {
siteIndexer = (SiteIndexer) iterator.next();
try {
@@ -1176,7 +1218,7 @@
}
long indexingElapsedTime = now - indexingStartTime;
- if (logger.isInfoEnabled()) {
+ if (logger.isInfoEnabled() && indexOrderCount>0 ) {
logger.info(
"Finished processing " + indexOrderCount +
" indexing orders in " + indexingElapsedTime + "ms.");
@@ -1189,8 +1231,8 @@
// FIXME : oops, As we are in a separate thread, this would may
have sence
// to terminate Connection ?
- org.jahia.services.database.ConnectionDispenser.
- terminateConnection ();
+ //org.jahia.services.database.ConnectionDispenser.
+ // terminateConnection ();
int size = 0;
synchronized (this) {
@@ -1355,7 +1397,7 @@
doc.add (Field.Keyword (indObj.getKeyFieldName (),
indObj.getKey ()));
doc.add(Field.UnStored(JahiaSearchConstant.FIELD_TEXT,
freeText.toString()));
-
+
return doc;
}
@@ -1439,6 +1481,7 @@
String.valueOf (indObj.getKey ()));
removeDoc (indObj.getSiteId (), term, reader);
+ //this.ramIndexer.removeDoc(term,indObj.getSiteId());
}
//--------------------------------------------------------------------------
@@ -1827,6 +1870,24 @@
closeIndexReader(reader);
}
+ RAMDirectory ramDir = new RAMDirectory();
+ IndexWriter ramWriter = new
IndexWriter(ramDir,indexAnalyzer,true);
+ ramWriter.mergeFactor = 50;
+ ramWriter.minMergeDocs = 1000;
+ luceneDoc = null;
+ size = luceneDocs.size();
+ for ( int i=0; i<size; i++ ){
+ luceneDoc = (Document)luceneDocs.get(i);
+ if (luceneDoc != null){
+ ramWriter.addDocument(luceneDoc);
+ }
+ }
+
+ luceneDocs = null;
+ docs = null;
+
+ ramWriter.close();
+
IndexWriter fsWriter = null;
try {
fsWriter = getIndexWriter(siteId,indexAnalyzer,false);
@@ -1843,6 +1904,7 @@
fsWriter.mergeFactor = 30;
}
}
+ /*
luceneDoc = null;
size = luceneDocs.size();
for ( int i=0; i<size; i++ ){
@@ -1850,12 +1912,16 @@
if (luceneDoc != null){
fsWriter.addDocument(luceneDoc);
}
- }
+ }*/
+ fsWriter.addIndexes(new Directory[]{ramDir});
}
} catch ( Throwable t ) {
logger.debug("Error adding doc from index", t);
} finally {
closeIndexWriter(fsWriter);
+ ramDir = null;
+ ramWriter = null;
+ fsWriter = null;
}
} else if ( this.getLastOperation() == REMOVE ) {
IndexReader reader = getIndexReader(siteId);
@@ -1867,6 +1933,7 @@
doc = null;
}
closeIndexReader(reader);
+ reader = null;
}
this.docs = new Vector();
this.setLastOperation(UNDEFINED);
Index: RAMIndexer.java
===================================================================
RCS file:
/cvs/jahia/src/java/org/jahia/services/search/Attic/RAMIndexer.java,v
retrieving revision 1.1.2.3
retrieving revision 1.1.2.4
diff -u -r1.1.2.3 -r1.1.2.4
--- RAMIndexer.java 3 Jun 2005 10:29:20 -0000 1.1.2.3
+++ RAMIndexer.java 8 Jun 2005 20:53:45 -0000 1.1.2.4
@@ -39,7 +39,7 @@
public void run(){
while ( !killed ){
try {
- Thread.sleep(60000);
+ Thread.sleep(30000);
long now = System.currentTimeMillis();
synchronized(this){
Iterator iterator = docs.keySet().iterator();
@@ -47,14 +47,16 @@
Vector v = null;
while ( iterator.hasNext() ){
I = (Integer)iterator.next();
+ this.closeIndexWriter(I.intValue());
v = (Vector)docs.get(I);
int size = v.size();
TempDocument doc = null;
Vector newV = new Vector();
for ( int i=0; i<size; i++ ){
doc = (TempDocument)v.get(i);
- if ( (now-doc.getCreationDate())<60000){
+ if ( (now-doc.getCreationDate())<30000){
newV.add(doc);
+ this.addDoc(doc.getDoc(),I.intValue());
}
}
docs.put(I,newV);
@@ -79,6 +81,33 @@
thread.start();
}
+ public synchronized void removeDoc(Term term, int siteId)
+ {
+ Directory dir = (Directory)this.ramDirs.get(new Integer(siteId));
+ if ( dir != null ){
+ try {
+ IndexWriter writer = (IndexWriter)this.indexes.get(new
Integer(siteId));
+ if ( writer != null ){
+ try {
+ writer.close();
+ } catch ( Throwable t){
+ }
+ }
+ try {
+ IndexReader reader = IndexReader.open(dir);
+ reader.delete(term);
+ writer = new IndexWriter(dir, analyzer, false);
+ } catch ( Throwable t ){
+ }
+ if ( writer != null ){
+ this.indexes.put(new Integer(siteId),writer);
+ }
+ } catch ( Throwable t){
+ logger.debug("Exception removing term from RAMIndexer",t);
+ }
+ }
+ }
+
public synchronized void addDoc(Document doc, int siteId){
if ( this.killed || doc == null ){
return;
@@ -88,8 +117,9 @@
docCount = writer.docCount();
if ( docCount > maxDoc ){
try {
+ closeIndexWriter(siteId);
Directory newDir = new RAMDirectory();
- writer = new IndexWriter(new RAMDirectory(),analyzer,true);
+ writer = new IndexWriter(newDir,analyzer,true);
writer.minMergeDocs = 1000;
writer.mergeFactor = 50;
writer.setUseCompoundFile(false);
@@ -97,6 +127,8 @@
for ( int i=0; i<50; i++ ){
if ( v.size() > 50 ){
v.remove(0);
+ } else {
+ break;
}
}
int size = v.size();
@@ -120,14 +152,18 @@
TempDocument tempDoc = new TempDocument(doc);
v.add(tempDoc);
writer.addDocument(doc);
- writer.close();
-
+ try {
+ writer.close();
+ } catch ( Throwable t) {
+ }
RAMDirectory oldDirectory = (RAMDirectory)this.ramDirs.get(new
Integer(siteId));
- writer = new IndexWriter(oldDirectory,analyzer,false);
- writer.mergeFactor = 50;
- writer.minMergeDocs = 1000;
- writer.setUseCompoundFile(false);
- this.indexes.put(new Integer(siteId),writer);
+ try {
+ writer = new IndexWriter(oldDirectory, analyzer, false);
+ } catch ( Throwable t ){
+ }
+ if ( writer != null ){
+ this.indexes.put(new Integer(siteId),writer);
+ }
} catch ( Throwable t) {
logger.debug("Exception adding doc to RAMIndexer",t);
@@ -147,23 +183,26 @@
}
private IndexWriter getIndexWriter(int siteId){
- synchronized(this.indexes){
- IndexWriter writer = (IndexWriter) this.indexes.get(new
Integer(siteId));
- if ( writer == null ) {
- RAMDirectory ramDir = new RAMDirectory();
- try {
- writer = new IndexWriter(ramDir,analyzer,true);
- writer.minMergeDocs = 1000;
- writer.mergeFactor = 50;
- writer.setUseCompoundFile(false);
- this.indexes.put(new Integer(siteId),writer);
- this.ramDirs.put(new Integer(siteId),ramDir);
- } catch ( Throwable t) {
- logger.debug("Exception creatring IndexWriter for
RAMIndexer",t);
- }
+ IndexWriter writer = (IndexWriter) this.indexes.get(new
Integer(siteId));
+ if ( writer == null ) {
+ RAMDirectory ramDir = new RAMDirectory();
+ try {
+ writer = new IndexWriter(ramDir,analyzer,true);
+ writer.minMergeDocs = 1000;
+ writer.mergeFactor = 50;
+ writer.setUseCompoundFile(false);
+ this.indexes.put(new Integer(siteId),writer);
+ this.ramDirs.put(new Integer(siteId),ramDir);
+ } catch ( Throwable t) {
+ logger.debug("Exception creatring IndexWriter for
RAMIndexer",t);
}
- return writer;
}
+ return writer;
+ }
+
+ private void closeIndexWriter(int siteId){
+ this.indexes.remove(new Integer(siteId));
+ this.ramDirs.remove(new Integer(siteId));
}
public void shutDown(){
Index: RemovedField.java
===================================================================
RCS file:
/cvs/jahia/src/java/org/jahia/services/search/Attic/RemovedField.java,v
retrieving revision 1.3.4.2
retrieving revision 1.3.4.2.2.1
diff -u -r1.3.4.2 -r1.3.4.2.2.1
--- RemovedField.java 26 Aug 2004 16:29:41 -0000 1.3.4.2
+++ RemovedField.java 8 Jun 2005 20:53:45 -0000 1.3.4.2.2.1
@@ -21,19 +21,21 @@
* @param key
* @param fields
*/
+ /*
public RemovedField (int siteId, String keyFieldName,
String key, Hashtable fields) {
super (siteId, keyFieldName, key, fields);
- }
+ }*/
/**
* @param siteId
* @param key
* @param fields
*/
+ /*
public RemovedField (int siteId, String key, Hashtable fields) {
super (siteId, null, key, fields);
- }
+ }*/
public RemovedField (JahiaField theField) {
Index: ScheduledSiteIndexationJob.java
===================================================================
RCS file:
/cvs/jahia/src/java/org/jahia/services/search/Attic/ScheduledSiteIndexationJob.java,v
retrieving revision 1.1.2.7
retrieving revision 1.1.2.8
diff -u -r1.1.2.7 -r1.1.2.8
--- ScheduledSiteIndexationJob.java 3 Jun 2005 10:29:20 -0000 1.1.2.7
+++ ScheduledSiteIndexationJob.java 8 Jun 2005 20:53:45 -0000 1.1.2.8
@@ -82,6 +82,7 @@
JahiaField aField;
int size = listFieldID.size();
status.setNbDocsToIndex(size);
+ status.setNbIndexedDocs(0);
boolean success = true;
for (int i = 0; i < size; i++) {
if ( status.isShutdown() ){
Index: SearchEngine.java
===================================================================
RCS file:
/cvs/jahia/src/java/org/jahia/services/search/Attic/SearchEngine.java,v
retrieving revision 1.17.2.1.2.4
retrieving revision 1.17.2.1.2.5
diff -u -r1.17.2.1.2.4 -r1.17.2.1.2.5
--- SearchEngine.java 3 Jun 2005 16:54:46 -0000 1.17.2.1.2.4
+++ SearchEngine.java 8 Jun 2005 20:53:45 -0000 1.17.2.1.2.5
@@ -113,7 +113,10 @@
if ( multiSearcher != null ){
hits = multiSearcher.search(query);
}
-
+ try {
+ multiSearcher.close();
+ } catch (Throwable t){
+ }
Vector parsedObjects = getParsedObjects(hits);
if ( parsedObjects == null ){
parsedObjects = new Vector();
Index: SiteIndexationStatus.java
===================================================================
RCS file:
/cvs/jahia/src/java/org/jahia/services/search/Attic/SiteIndexationStatus.java,v
retrieving revision 1.1.2.1
retrieving revision 1.1.2.2
diff -u -r1.1.2.1 -r1.1.2.2
--- SiteIndexationStatus.java 3 Jun 2005 10:29:20 -0000 1.1.2.1
+++ SiteIndexationStatus.java 8 Jun 2005 20:53:45 -0000 1.1.2.2
@@ -95,19 +95,19 @@
}
}
- public int getNbDocsToIndex() {
+ public synchronized int getNbDocsToIndex() {
return nbDocsToIndex;
}
- public void setNbDocsToIndex(int nbDocsToIndex) {
+ public synchronized void setNbDocsToIndex(int nbDocsToIndex) {
this.nbDocsToIndex = nbDocsToIndex;
}
- public int getNbIndexedDocs() {
+ public synchronized int getNbIndexedDocs() {
return nbIndexedDocs;
}
- public void setNbIndexedDocs(int nbIndexedDocs) {
+ public synchronized void setNbIndexedDocs(int nbIndexedDocs) {
this.nbIndexedDocs = nbIndexedDocs;
}
