Author: toad
Date: 2008-02-28 22:24:09 +0000 (Thu, 28 Feb 2008)
New Revision: 18233
Modified:
trunk/plugins/XMLSpider/XMLSpider.java
Log:
27:
- Reduce max retries to 9 so that we only go onto the cooldown queue 3 times
not 4.
- Increase max parallel requests to 500 to try to compensate for the cooldown
queue. Hopefully memory won't be a problem ... you need a lot of RAM to run a
spider anyway...
Modified: trunk/plugins/XMLSpider/XMLSpider.java
===================================================================
--- trunk/plugins/XMLSpider/XMLSpider.java 2008-02-28 22:20:25 UTC (rev
18232)
+++ trunk/plugins/XMLSpider/XMLSpider.java 2008-02-28 22:24:09 UTC (rev
18233)
@@ -143,7 +143,7 @@
public Set allowedMIMETypes;
private static final int MAX_ENTRIES = 2000;
private static final long MAX_SUBINDEX_UNCOMPRESSED_SIZE = 256*1024;
- private static int version = 26;
+ private static int version = 27;
private static final String pluginName = "XML spider "+version;
/**
* Gives the allowed fraction of total time spent on generating indices
with
@@ -161,7 +161,7 @@
// private final HashMap positionsByWordByURI = new HashMap(); /* String
(URI) -> HashMap (String (word) -> Integer[] (Positions)) */
private final HashMap positionsByWordById = new HashMap();
// Can have many; this limit only exists to save memory.
- private static final int maxParallelRequests = 100;
+ private static final int maxParallelRequests = 500;
private int maxShownURIs = 15;
private NodeClientCore core;
@@ -707,8 +707,8 @@
this.core = pluginManager.getClientCore();
this.ctx = core.makeClient((short) 0).getFetchContext();
- ctx.maxSplitfileBlockRetries = 10;
- ctx.maxNonSplitfileRetries = 10;
+ ctx.maxSplitfileBlockRetries = 9; // Will be on the cooldown
queue 3 times.
+ ctx.maxNonSplitfileRetries = 9; // Will be on the cooldown
queue 3 times.
ctx.maxTempLength = 2 * 1024 * 1024;
ctx.maxOutputLength = 2 * 1024 * 1024;
allowedMIMETypes = new HashSet();
