Author: j16sdiz
Date: 2008-12-17 05:37:12 +0000 (Wed, 17 Dec 2008)
New Revision: 24403
Modified:
trunk/plugins/XMLSpider/XMLSpider.java
Log:
retry r24379 with deadlock fix
Modified: trunk/plugins/XMLSpider/XMLSpider.java
===================================================================
--- trunk/plugins/XMLSpider/XMLSpider.java 2008-12-17 05:37:03 UTC (rev
24402)
+++ trunk/plugins/XMLSpider/XMLSpider.java 2008-12-17 05:37:12 UTC (rev
24403)
@@ -169,10 +169,7 @@
private Vector<String> indices;
private int match;
private long time_taken;
-/*
- * minTimeBetweenEachIndexRewriting in seconds
- */
- private static final int minTimeBetweenEachIndexRewriting = 600;
+
/**
* directory where the generated indices are stored.
* Needs to be created before it can be used
@@ -229,7 +226,6 @@
* Adds the found uri to the list of to-be-retrieved uris. <p>Every usk
uri added as ssk.
* @param uri the new uri that needs to be fetched for further indexing
*/
- // synchronized: this->page
public void queueURI(FreenetURI uri, String comment, boolean force) {
String sURI = uri.toString();
for (String ext : BADLIST_EXTENSTION)
@@ -273,8 +269,9 @@
ArrayList<ClientGetter> toStart = null;
synchronized (this) {
- if (stopped)
+ if (stopped)
return;
+ synchronized (runningFetch) {
int running = runningFetch.size();
@@ -309,6 +306,7 @@
db.store(page);
}
}
+ }
}
for (ClientGetter g : toStart) {
@@ -322,7 +320,6 @@
}
}
-
private class MyClientCallback implements ClientCallback {
final Page page;
@@ -421,10 +418,10 @@
}
} finally {
synchronized (this) {
+ runningFetch.remove(page);
page.lastChange = System.currentTimeMillis();
db.store(page);
db.commit();
- runningFetch.remove(page);
}
startSomeRequests();
}
@@ -433,11 +430,11 @@
public void onFailure(FetchException fe, ClientGetter state, Page page)
{
Logger.minor(this, "Failed: " + page + " : " + state, fe);
- try {
synchronized (this) {
if (stopped)
return;
+ synchronized (page) {
if (fe.newURI != null) {
// redirect, mark as succeeded
queueURI(fe.newURI, "redirect from " +
state.getURI(), false);
@@ -457,13 +454,12 @@
db.store(page);
}
+ }
+ runningFetch.remove(page);
}
startSomeRequests();
- } finally {
- runningFetch.remove(page);
- }
- }
+ }
/**
* generates the main index file that can be used by librarian for
searching in the list of
@@ -474,7 +470,7 @@
* @throws IOException
* @throws NoSuchAlgorithmException
*/
- private synchronized void makeMainIndex() throws
IOException,NoSuchAlgorithmException {
+ private void makeMainIndex() throws IOException,
NoSuchAlgorithmException {
// Produce the main index file.
Logger.minor(this, "Producing top index...");
@@ -591,7 +587,7 @@
* of the hash code of the words
* @throws Exception
*/
- private synchronized void makeSubIndices() throws Exception{
+ private void makeSubIndices() throws Exception {
Logger.normal(this, "Generating index...");
Query query = db.query();
@@ -632,7 +628,7 @@
}
- private synchronized void generateSubIndex(int p, List<String> list)
throws Exception {
+ private void generateSubIndex(int p, List<String> list) throws
Exception {
boolean logMINOR = Logger.shouldLog(Logger.MINOR, this);
/*
* if the list is less than max allowed entries in a file then
directly generate the xml
@@ -687,7 +683,7 @@
* @param prefix number of matching bits of md5
* @throws Exception
*/
- protected synchronized void generateXML(List<String> list, int prefix)
throws TooBigIndexException, Exception
+ protected void generateXML(List<String> list, int prefix) throws
TooBigIndexException, Exception
{
String p = list.get(0).substring(0, prefix);
indices.add(p);
@@ -1030,7 +1026,6 @@
Thread.sleep(30 * 1000); // Let the
node start up
} catch (InterruptedException e){}
startSomeRequests();
- scheduleMakeIndex();
}
}, "Spider Plugin Starter");
}
@@ -1078,6 +1073,18 @@
HTMLNode pageNode = pageMaker.getPageNode(pluginName, null);
HTMLNode contentNode = pageMaker.getContentNode(pageNode);
+ if (request.isPartSet("createIndex")) {
+ synchronized (this) {
+ if (!writingIndex) {
+ scheduleMakeIndex();
+
+ HTMLNode infobox =
pageMaker.getInfobox("infobox infobox-success", "Scheduled Creating Index");
+ infobox.addChild("%", "Index will start
create soon.");
+ contentNode.addChild(infobox);
+ }
+ }
+ }
+
String addURI = request.getPartAsString("addURI", 512);
if (addURI != null && addURI.length() != 0) {
try {
@@ -1133,17 +1140,33 @@
nextTableCell = overviewTableRow.addChild("td", "class",
"second");
HTMLNode mainBox = pageMaker.getInfobox("Main");
HTMLNode mainContent = pageMaker.getContentNode(mainBox);
- HTMLNode form = mainContent.addChild("form", //
+ HTMLNode addForm = mainContent.addChild("form", //
new String[] { "action", "method" }, //
new String[] { "plugins.XMLSpider.XMLSpider", "post" });
- form.addChild("label", "for", "addURI", "Add URI:");
- form.addChild("input", new String[] { "name", "style" }, new
String[] { "addURI", "width: 20em;" });
- form.addChild("input", //
+ addForm.addChild("label", "for", "addURI", "Add URI:");
+ addForm.addChild("input", new String[] { "name", "style" }, new
String[] { "addURI", "width: 20em;" });
+ addForm.addChild("input", //
new String[] { "name", "type", "value" },//
new String[] { "formPassword", "hidden",
core.formPassword });
- form.addChild("input", "type", "submit");
+ addForm.addChild("input", "type", "submit");
nextTableCell.addChild(mainBox);
+ HTMLNode indexBox = pageMaker.getInfobox("Create Index");
+ HTMLNode indexContent = pageMaker.getContentNode(indexBox);
+ HTMLNode indexForm = indexContent.addChild("form", //
+ new String[] { "action", "method" }, //
+ new String[] { "plugins.XMLSpider.XMLSpider", "post" });
+ indexForm.addChild("input", //
+ new String[] { "name", "type", "value" },//
+ new String[] { "formPassword", "hidden",
core.formPassword });
+ indexForm.addChild("input", //
+ new String[] { "name", "type", "value" },//
+ new String[] { "createIndex", "hidden", "createIndex"
});
+ indexForm.addChild("input", //
+ new String[] { "type", "value" }, //
+ new String[] { "submit", "Create Index Now" });
+ nextTableCell.addChild(indexBox);
+
HTMLNode runningBox = pageMaker.getInfobox("Running URI");
runningBox.addAttribute("style", "right: 0;");
HTMLNode runningContent = pageMaker.getContentNode(runningBox);
@@ -1298,8 +1321,6 @@
tProducedIndex = System.currentTimeMillis();
} finally {
- if (!stopped)
- scheduleMakeIndex();
synchronized (this) {
writingIndex = false;
notifyAll();
@@ -1321,7 +1342,7 @@
return NativeThread.LOW_PRIORITY;
}
- }, minTimeBetweenEachIndexRewriting * 1000);
+ }, 1);
}
public void onFoundEdition(long l, USK key){
_______________________________________________
cvs mailing list
[email protected]
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
