Author: j16sdiz
Date: 2008-12-29 01:17:49 +0000 (Mon, 29 Dec 2008)
New Revision: 24808
Modified:
trunk/plugins/XMLSpider/XMLSpider.java
Log:
limit the query with timestamp, hope to save some time and memory
Modified: trunk/plugins/XMLSpider/XMLSpider.java
===================================================================
--- trunk/plugins/XMLSpider/XMLSpider.java 2008-12-28 15:10:39 UTC (rev
24807)
+++ trunk/plugins/XMLSpider/XMLSpider.java 2008-12-29 01:17:49 UTC (rev
24808)
@@ -153,7 +153,8 @@
}
protected List<Page> queuedRequestCache = new ArrayList<Page>();
-
+ protected long lastPrefetchedTimeStamp = -1;
+
public void startSomeRequests() {
ArrayList<ClientGetter> toStart = null;
synchronized (this) {
@@ -170,14 +171,30 @@
Query query = db.query();
query.constrain(Page.class);
query.descend("status").constrain(Status.QUEUED);
+ if (lastPrefetchedTimeStamp != -1) {
+
query.descend("lastChange").constrain(lastPrefetchedTimeStamp - 1000).greater();
+
query.descend("lastChange").constrain(lastPrefetchedTimeStamp + 1800 *
1000).smaller();
+ }
query.descend("lastChange").orderAscending();
@SuppressWarnings("unchecked")
ObjectSet<Page> queuedSet =
query.execute();
+
+
System.out.println("lastPrefetchedTimeStamp=" + lastPrefetchedTimeStamp + ",
BLAR = "
+ + queuedSet.size());
+ if (lastPrefetchedTimeStamp != -1 &&
queuedSet.isEmpty()) {
+ lastPrefetchedTimeStamp = -1;
+ startSomeRequests();
+ return;
+ }
while (queuedRequestCache.size() <
config.getMaxParallelRequests() * 2 && queuedSet.hasNext()) {
Page page = queuedSet.next();
- if
(!runningFetch.containsKey(page))
+ if
(!runningFetch.containsKey(page)) {
queuedRequestCache.add(page);
+
+ if (page.lastChange >
lastPrefetchedTimeStamp)
+
lastPrefetchedTimeStamp = page.lastChange;
+ }
}
}
_______________________________________________
cvs mailing list
[email protected]
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
