Author: j16sdiz
Date: 2008-12-26 10:07:04 +0000 (Fri, 26 Dec 2008)
New Revision: 24793
Modified:
trunk/plugins/XMLSpider/XMLSpider.java
Log:
perfetch more pages for start
Modified: trunk/plugins/XMLSpider/XMLSpider.java
===================================================================
--- trunk/plugins/XMLSpider/XMLSpider.java 2008-12-26 10:06:51 UTC (rev
24792)
+++ trunk/plugins/XMLSpider/XMLSpider.java 2008-12-26 10:07:04 UTC (rev
24793)
@@ -165,6 +165,7 @@
if (running >= config.getMaxParallelRequests())
return;
+ // prefetch 2 * config.getMaxParallelRequests()
entries
if (queuedRequestCache.isEmpty()) {
Query query = db.query();
query.constrain(Page.class);
@@ -173,16 +174,15 @@
@SuppressWarnings("unchecked")
ObjectSet<Page> queuedSet =
query.execute();
- for (int i = 0 ;
- i <
config.getMaxParallelRequests() * 2 && queuedSet.hasNext();
- i++) { // cache 2 *
maxParallelRequests
-
queuedRequestCache.add(queuedSet.next());
+ while (queuedRequestCache.size() <
config.getMaxParallelRequests() * 2 && queuedSet.hasNext()) {
+ Page page = queuedSet.next();
+ if
(!runningFetch.containsKey(page))
+
queuedRequestCache.add(page);
}
}
-
queuedRequestCache.removeAll(runningFetch.keySet());
+ // perpare to start
toStart = new
ArrayList<ClientGetter>(config.getMaxParallelRequests() - running);
-
Iterator<Page> it =
queuedRequestCache.iterator();
while (running + toStart.size() <
config.getMaxParallelRequests() && it.hasNext()) {
_______________________________________________
cvs mailing list
[email protected]
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
