Author: knguyen
Date: Wed Jun 20 17:06:06 2007
New Revision: 17673
URL: https://svndev.jahia.net/websvn/listing.php?sc=3D1&rev=3D17673&repname=
=3Djahia
Log:
PEU-50: search indexation optimization
Modified:
branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/hibernate/model/in=
dexingjob/JahiaContainerIndexingJob.java
branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/Do=
cumentField.java
branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/Ja=
hiaSearchBaseService.java
branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/Ja=
hiaSiteIndexingJob.java
branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/va=
lves/ContainerSearchIndexProcessValveImpl.java
Modified: branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/hibernate/mo=
del/indexingjob/JahiaContainerIndexingJob.java
URL: https://svndev.jahia.net/websvn/diff.php?path=3D/branches/JAHIA-5-0-SP=
-BRANCH/core/src/java/org/jahia/hibernate/model/indexingjob/JahiaContainerI=
ndexingJob.java&rev=3D17673&repname=3Djahia
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D
--- branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/hibernate/model/in=
dexingjob/JahiaContainerIndexingJob.java (original)
+++ branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/hibernate/model/in=
dexingjob/JahiaContainerIndexingJob.java Wed Jun 20 17:06:06 2007
@@ -28,6 +28,7 @@
import java.util.List;
import java.util.Iterator;
import java.util.HashMap;
+import java.util.ArrayList;
=
/**
* Created by IntelliJ IDEA.
@@ -108,8 +109,9 @@
.getIndexableDocumentsForContainer(this.getCtnId().intValu=
e(), user);
Iterator iterator =3D docs.iterator();
IndexableDocument doc =3D null;
- HashMap fieldDocs =3D null;
- HashMap removeFieldDocs =3D null;
+ ArrayList processedFieldIds =3D new ArrayList();
+ HashMap fieldDocs =3D new HashMap();
+ HashMap removeFieldDocs =3D new HashMap();
while ( iterator.hasNext() ){
doc =3D (IndexableDocument)iterator.next();
if ( doc instanceof RemovableDocument ){
@@ -117,22 +119,33 @@
} else {
toAdd.add(doc);
}
- fieldDocs =3D new HashMap();
- removeFieldDocs =3D new HashMap();
- List childDocs =3D doc.getChildIndexableDocuments();
- Iterator childDocsIterator =3D childDocs.iterator();
- IndexableDocument childDoc =3D null;
- while ( childDocsIterator.hasNext() ){
- childDoc =3D (IndexableDocument)childDocsIterator.next();
- if ( childDoc instanceof RemovableDocument ){
- if ( !removeFieldDocs.containsKey(childDoc.getKey()) ){
- toRemove.add(childDoc);
- removeFieldDocs.put(childDoc.getKey(),childDoc.get=
Key());
- }
- } else if ( childDoc instanceof JahiaFieldIndexableDocumen=
t){
- if ( !fieldDocs.containsKey(childDoc.getKey()) ){
- toAdd.add(childDoc);
- fieldDocs.put(childDoc.getKey(),childDoc.getKey());
+ List childFields =3D doc.getChildIndexableDocuments();
+ Iterator it =3D childFields.iterator();
+ Integer fieldId =3D null;
+ while ( it.hasNext() ){
+ fieldId =3D (Integer)it.next();
+ if ( !processedFieldIds.contains(fieldId) ){
+ processedFieldIds.add(fieldId);
+ try {
+ List fieldChildDocs =3D ServicesRegistry.getInstan=
ce().getJahiaSearchService()
+ .getIndexableDocumentsForField(fieldId.int=
Value(),user);
+ Iterator childDocsIterator =3D fieldChildDocs.iter=
ator();
+ IndexableDocument childDoc =3D null;
+ while ( childDocsIterator.hasNext() ){
+ childDoc =3D (IndexableDocument)childDocsItera=
tor.next();
+ if ( childDoc instanceof RemovableDocument ){
+ if ( !removeFieldDocs.containsKey(childDoc=
.getKey()) ){
+ toRemove.add(childDoc);
+ removeFieldDocs.put(childDoc.getKey(),=
childDoc.getKey());
+ }
+ } else if ( childDoc instanceof JahiaFieldInde=
xableDocument){
+ if ( !fieldDocs.containsKey(childDoc.getKe=
y()) ){
+ toAdd.add(childDoc);
+ fieldDocs.put(childDoc.getKey(),childD=
oc.getKey());
+ }
+ }
+ }
+ } catch ( Throwable t ){
}
}
}
Modified: branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/sea=
rch/DocumentField.java
URL: https://svndev.jahia.net/websvn/diff.php?path=3D/branches/JAHIA-5-0-SP=
-BRANCH/core/src/java/org/jahia/services/search/DocumentField.java&rev=3D17=
673&repname=3Djahia
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D
--- branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/Do=
cumentField.java (original)
+++ branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/Do=
cumentField.java Wed Jun 20 17:06:06 2007
@@ -57,9 +57,11 @@
=
private boolean unstoredText(String fieldName){
if ( JahiaSearchConstant.ALL_FULLTEXT_SEARCH_FIELD.equals(fieldNam=
e)
+ || JahiaSearchConstant.ALL_FULLTEXT_SEARCH_FIELD_FOR_QUERY=
_REWRITE.equals(fieldName)
|| JahiaSearchConstant.CONTENT_FULLTEXT_SEARCH_FIELD.equal=
s(fieldName)
|| JahiaSearchConstant.FILE_CONTENT_FULLTEXT_SEARCH_FIELD.=
equals(fieldName)
|| JahiaSearchConstant.CONTENT_FULLTEXT_SEARCH_FIELD_FOR_Q=
UERY_REWRITE.equals(fieldName)
+ || JahiaSearchConstant.METADATA_FULLTEXT_SEARCH_FIELD_FOR_=
QUERY_REWRITE.equals(fieldName)
|| JahiaSearchConstant.METADATA_FULLTEXT_SEARCH_FIELD.equa=
ls(fieldName) ){
return true;
}
Modified: branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/sea=
rch/JahiaSearchBaseService.java
URL: https://svndev.jahia.net/websvn/diff.php?path=3D/branches/JAHIA-5-0-SP=
-BRANCH/core/src/java/org/jahia/services/search/JahiaSearchBaseService.java=
&rev=3D17673&repname=3Djahia
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D
--- branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/Ja=
hiaSearchBaseService.java (original)
+++ branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/Ja=
hiaSearchBaseService.java Wed Jun 20 17:06:06 2007
@@ -584,6 +584,7 @@
} catch ( InterruptedException int=
e ){
}
} else {
+ notifySiteIndexingJobOfBatchingInd=
exation(toRemove,toAdd);
searchHandler.batchIndexing(toRemo=
ve,toAdd);
continue;
}
@@ -650,6 +651,29 @@
}
}
=
+ protected void notifySiteIndexingJobOfBatchingIndexation(List toRemove=
, List toAdd) {
+ try {
+ List executingJobs =3D ServicesRegistry.getInstance().getSched=
ulerService()
+ .getCurrentlyExecutingRamJobs();
+ if ( executingJobs =3D=3D null ){
+ return;
+ }
+ Iterator it =3D executingJobs.iterator();
+ JobExecutionContext jobExContext =3D null;
+ Job job =3D null;
+ while ( it.hasNext() ){
+ jobExContext =3D (JobExecutionContext)it.next();
+ job =3D jobExContext.getJobInstance();
+ if ( job !=3D null && job instanceof JahiaSiteIndexingJob =
){
+ ((JahiaSiteIndexingJob)job).addConcurrentJobs(toRemove=
,false);
+ ((JahiaSiteIndexingJob)job).addConcurrentJobs(toAdd,tr=
ue);
+ }
+ }
+ } catch ( Throwable t ){
+ logger.debug("Error notifying jahiasite indexing jobs",t);
+ }
+ }
+
protected void wakeupSearchIndexer() {
Map searchHandlers =3D this.getSearchManager().getSearchHandlers();
Iterator it =3D searchHandlers.values().iterator();
@@ -1217,38 +1241,12 @@
if (! this.localIndexing ){
return;
}
+ indJob =3D new JahiaContainerIndexingJob(ctnId, System.cur=
rentTimeMillis());
+ List toAdd =3D new ArrayList();
+ List toRemove =3D new ArrayList();
+ indJob.prepareBatchIndexation(toRemove,toAdd,user);
JahiaSite site =3D sitesService.getSite(contentContainer.g=
etSiteID());
- List docs =3D getIndexableDocumentsForContainer(ctnId,user=
);
- Iterator iterator =3D docs.iterator();
- List childDocs =3D null;
- Iterator childDocsIterator =3D null;
- Map fieldDocs =3D null;
- Map removeFieldDocs =3D null;
- while ( iterator.hasNext() ){
- fieldDocs =3D new HashMap();
- removeFieldDocs =3D new HashMap();
- IndexableDocument doc =3D (IndexableDocument)iterator.=
next();
- this.getSearchHandler(site.getID())
- .addDocument(doc);
- childDocs =3D doc.getChildIndexableDocuments();
- childDocsIterator =3D childDocs.iterator();
- IndexableDocument childDoc =3D null;
- while ( childDocsIterator.hasNext() ){
- childDoc =3D (IndexableDocument)childDocsIterator.=
next();
- if ( childDoc instanceof RemovableDocument ){
- if ( !removeFieldDocs.containsKey(childDoc.get=
Key()) ){
- this.getSearchHandler(site.getID())
- .addDocument(childDoc);
- removeFieldDocs.put(childDoc.getKey(),chil=
dDoc.getKey());
- }
- } else if ( childDoc instanceof JahiaFieldIndexabl=
eDocument ){
- if ( !fieldDocs.containsKey(childDoc.getKey())=
){
- this.getSearchHandler(site.getID()).addDoc=
ument(childDoc);
- fieldDocs.put(childDoc.getKey(),childDoc.g=
etKey());
- }
- }
- }
- }
+ this.getSearchHandler(site.getID()).batchIndexing(toRemove=
,toAdd);
} else {
this.addIndexingJob(indJob);
}
Modified: branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/sea=
rch/JahiaSiteIndexingJob.java
URL: https://svndev.jahia.net/websvn/diff.php?path=3D/branches/JAHIA-5-0-SP=
-BRANCH/core/src/java/org/jahia/services/search/JahiaSiteIndexingJob.java&r=
ev=3D17673&repname=3Djahia
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D
--- branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/Ja=
hiaSiteIndexingJob.java (original)
+++ branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/Ja=
hiaSiteIndexingJob.java Wed Jun 20 17:06:06 2007
@@ -16,6 +16,7 @@
import org.jahia.services.scheduler.RamJob;
import org.jahia.hibernate.manager.SpringContextSingleton;
import org.jahia.hibernate.manager.JahiaContainerDefinitionManager;
+import org.jahia.hibernate.model.indexingjob.JahiaContainerIndexingJob;
import org.jahia.content.TreeOperationResult;
import org.jahia.content.NodeOperationResult;
import org.springframework.context.ApplicationContext;
@@ -48,6 +49,8 @@
=
private int siteId =3D -1;
=
+ private Map concurrentJobs =3D new HashMap();
+
public JahiaSiteIndexingJob () {
}
=
@@ -106,8 +109,10 @@
locales.add(Locale.ENGLISH);
EntryLoadRequest loadRequest =3D new EntryLoadRequest(EntryLoa=
dRequest.STAGING_WORKFLOW_STATE,0,locales);
loadRequest.setWithMarkedForDeletion(true);
- searchService.removeFromSearchEngine(siteId, JahiaSearchConsta=
nt.JAHIA_ID,
+ if ( lastProcessedPage =3D=3D -1 ){
+ searchService.removeFromSearchEngine(siteId, JahiaSearchCo=
nstant.JAHIA_ID,
NumberPadding.pad(siteId), user,false,false);
+ }
JahiaContainersService containerService =3D ServicesRegistry.g=
etInstance().getJahiaContainersService();
JahiaPageService pageService =3D ServicesRegistry.getInstance()
.getJahiaPageService();
@@ -170,7 +175,7 @@
} catch ( Throwable t ){
}
}
- searchService.getSearchHandler(siteId).batchIndexi=
ng(toRemove,toAdd);
+ this.processBachIndexation(toRemove,toAdd);
if ( ! saveLastProcessedPage(id) ){
return;
}
@@ -196,41 +201,13 @@
ctnEnum =3D ctnIds.elements();
while ( ctnEnum.hasMoreElements() ){
ctnId =3D (Integer)ctnEnum.nextElement();
+ JahiaContainerIndexingJob indJob =3D new JahiaCon=
tainerIndexingJob (ctnId.intValue(), 0);
try {
- docs =3D ServicesRegistry.getInstance().getJah=
iaSearchService()
- .getIndexableDocumentsForContainer(ctnId.i=
ntValue(), user);
+ indJob.prepareBatchIndexation(toRemove,toAdd,u=
ser);
} catch ( Throwable t ){
logger.debug("Exception preparing indexing doc=
ument for container " + ctnId, t);
continue;
}
- iterator =3D docs.iterator();
- while ( iterator.hasNext() ){
- doc =3D (IndexableDocument)iterator.next();
- if ( doc instanceof RemovableDocument ){
- toRemove.add(doc);
- } else {
- toAdd.add(doc);
- }
- fieldDocs =3D new HashMap();
- removeFieldDocs =3D new HashMap();
- List childDocs =3D doc.getChildIndexableDocume=
nts();
- Iterator childDocsIterator =3D childDocs.itera=
tor();
- IndexableDocument childDoc =3D null;
- while ( childDocsIterator.hasNext() ){
- childDoc =3D (IndexableDocument)childDocsI=
terator.next();
- if ( childDoc instanceof RemovableDocument=
){
- if ( !removeFieldDocs.containsKey(chil=
dDoc.getKey()) ){
- toRemove.add(childDoc);
- removeFieldDocs.put(childDoc.getKe=
y(),childDoc.getKey());
- }
- } else if ( childDoc instanceof JahiaField=
IndexableDocument){
- if ( !fieldDocs.containsKey(childDoc.g=
etKey()) ){
- toAdd.add(childDoc);
- fieldDocs.put(childDoc.getKey(),ch=
ildDoc.getKey());
- }
- }
- }
- }
if ( toAdd.size() > batchSize ){
synchronized (lock){
try {
@@ -238,7 +215,7 @@
} catch ( Throwable t ){
}
}
- searchService.getSearchHandler(siteId).batchIn=
dexing(toRemove,toAdd);
+ this.processBachIndexation(toRemove,toAdd);
if ( ! saveLastProcessedPage(id) ){
return;
}
@@ -248,7 +225,7 @@
}
}
}
- searchService.getSearchHandler(siteId).batchIndexing(toRemove,=
toAdd);
+ this.processBachIndexation(toRemove,toAdd);
jobDataMap.put(BackgroundJob.JOB_STATUS, BackgroundJob.STATUS_=
SUCCESSFUL);
jobDataMap.put(BackgroundJob.RESULT,result);
saveSiteProperty(BackgroundJob.RESULT,STATUS_SUCCESSFUL);
@@ -297,6 +274,83 @@
}
}
=
+ public void addConcurrentJobs(List jobs, boolean toAdd){
+ synchronized(this.concurrentJobs){
+ if ( toAdd ){
+ List toAddList =3D (List)this.concurrentJobs.get("toAdd");
+ if (toAddList =3D=3D null ){
+ toAddList =3D new ArrayList();
+ this.concurrentJobs.put("toAdd",toAddList);
+ }
+ Iterator it =3D jobs.iterator();
+ IndexableDocument doc =3D null;
+ while ( it.hasNext() ){
+ doc =3D (IndexableDocument)it.next();
+ toAddList.add(doc.getKeyFieldName()+"_"+doc.getKey());
+ }
+ } else {
+ List toRemoveList =3D (List)this.concurrentJobs.get("toRem=
ove");
+ if (toRemoveList =3D=3D null ){
+ toRemoveList =3D new ArrayList();
+ this.concurrentJobs.put("toRemove",toRemoveList);
+ }
+ Iterator it =3D jobs.iterator();
+ IndexableDocument doc =3D null;
+ while ( it.hasNext() ){
+ doc =3D (IndexableDocument)it.next();
+ toRemoveList.add(doc.getKeyFieldName()+"_"+doc.getKey(=
));
+ }
+ }
+ }
+ }
+
+ protected void processBachIndexation(ArrayList toRemove, ArrayList toA=
dd) throws Exception {
+ synchronized (this.concurrentJobs){
+ ArrayList remList =3D new ArrayList();
+ ArrayList addList =3D new ArrayList();
+ Iterator it =3D toRemove.iterator();
+ IndexableDocument doc =3D null;
+ List checkList =3D (List)this.concurrentJobs.get("toRemove");
+ if ( checkList =3D=3D null || checkList.isEmpty() ){
+ remList =3D toRemove;
+ } else {
+ while ( it.hasNext() ){
+ doc =3D (IndexableDocument)it.next();
+ if (!checkList.contains(doc.getKeyFieldName()+"_"+doc.=
getKey())){
+ remList.add(doc);
+ }
+ }
+ }
+ it =3D toAdd.iterator();
+ List toAddCheckList =3D (List)this.concurrentJobs.get("toAdd");
+ if ( toAddCheckList =3D=3D null ){
+ toAddCheckList =3D new ArrayList();
+ }
+ if ( (checkList =3D=3D null || checkList.isEmpty()) && (toAddC=
heckList.isEmpty()) ){
+ addList =3D toAdd;
+ } else {
+ while ( it.hasNext() ){
+ String key =3D null;
+ doc =3D (IndexableDocument)it.next();
+ if ( doc instanceof JahiaContainerIndexableDocument ) {
+ key =3D JahiaSearchConstant.OBJECT_KEY + "_" + ((J=
ahiaContainerIndexableDocument)doc).getObjectKey().toString();
+ } else if ( doc instanceof JahiaPageIndexableDocument =
){
+ key =3D JahiaSearchConstant.OBJECT_KEY + "_" + ((J=
ahiaPageIndexableDocument)doc).getObjectKey().toString();
+ } else if ( doc instanceof JahiaFieldIndexableDocument=
){
+ key =3D JahiaSearchConstant.FIELD_FIELDID + "_" + =
doc.getField(JahiaSearchConstant.FIELD_FIELDID).getValues().get(0).toString=
();
+ }
+ if (!checkList.contains(key.toLowerCase()) && !toAddCh=
eckList.contains(key.toLowerCase())){
+ addList.add(doc);
+ }
+ }
+ }
+
+ ServicesRegistry.getInstance()
+ .getJahiaSearchService().getSearchHandler(siteId).batchInd=
exing(remList,addList);
+ this.concurrentJobs.clear();
+ }
+ }
+
protected void copyData(Properties props, JobDataMap data, String data=
Name,
String propName, String defaultValue){
String value =3D null;
Modified: branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/sea=
rch/valves/ContainerSearchIndexProcessValveImpl.java
URL: https://svndev.jahia.net/websvn/diff.php?path=3D/branches/JAHIA-5-0-SP=
-BRANCH/core/src/java/org/jahia/services/search/valves/ContainerSearchIndex=
ProcessValveImpl.java&rev=3D17673&repname=3Djahia
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D
--- branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/va=
lves/ContainerSearchIndexProcessValveImpl.java (original)
+++ branches/JAHIA-5-0-SP-BRANCH/core/src/java/org/jahia/services/search/va=
lves/ContainerSearchIndexProcessValveImpl.java Wed Jun 20 17:06:06 2007
@@ -178,11 +178,13 @@
DAVFileAccess file =3D JahiaWebdavBaseService.getInsta=
nce ()
.getDAVFileAccess (site, fField.getRealName ()=
);
if (file.isValid () && file.hasRevisions () && !file.i=
sCollection ()) {
+ /*
List docs =3D ServicesRegistry.getInstance().getJa=
hiaSearchService()
.getIndexableDocumentsForField(field.getID=
(),context.getUser());
if ( docs !=3D null && !docs.isEmpty() ){
doc.getChildIndexableDocuments().addAll(docs);
- }
+ }*/
+ doc.getChildIndexableDocuments().add(new Integer(f=
ield.getID()));
/*
ServicesRegistry.getInstance().getJahiaSearchServi=
ce()
.indexField(field.getID(), context.getUser(), fals=
e, true);
_______________________________________________
cvs_list mailing list
[email protected]
http://lists.jahia.org/cgi-bin/mailman/listinfo/cvs_list