Author: j16sdiz
Date: 2008-12-18 06:17:21 +0000 (Thu, 18 Dec 2008)
New Revision: 24513
Modified:
trunk/plugins/XMLSpider/XMLSpider.java
Log:
and update the old class (the commit hook don't allow me doing both in one go)
Modified: trunk/plugins/XMLSpider/XMLSpider.java
===================================================================
--- trunk/plugins/XMLSpider/XMLSpider.java 2008-12-18 06:17:11 UTC (rev
24512)
+++ trunk/plugins/XMLSpider/XMLSpider.java 2008-12-18 06:17:21 UTC (rev
24513)
@@ -8,11 +8,9 @@
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collections;
@@ -92,96 +90,12 @@
*
*/
public class XMLSpider implements FredPlugin, FredPluginHTTP,
FredPluginThreadless, FredPluginVersioned, USKCallback {
- static enum Status {
- /** For simplicity, running is also mark as QUEUED */
- QUEUED, SUCCEEDED, FAILED
- };
-
- static class MaxPageId {
- volatile long v;
-
- MaxPageId() {
- }
-
- MaxPageId(long v) {
- this.v = v;
- }
- }
-
public synchronized long getNextPageId() {
- long x = ++(maxPageId.v);
+ long x = maxPageId.incrementAndGet();
db.store(maxPageId);
return x;
}
- static class Page {
- /** Page Id */
- long id;
- /** URI of the page */
- String uri;
- /** Title */
- String pageTitle;
- /** Status */
- Status status = Status.QUEUED;
- /** Last Change Time */
- long lastChange = System.currentTimeMillis();
- /** Comment, for debugging */
- String comment;
-
- public Page() {} // for db4o callConstructors(true)
-
- @Override
- public int hashCode() {
- return (int) (id ^ (id >>> 32));
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
-
- return id == ((Page) obj).id;
- }
-
- @Override
- public String toString() {
- return "[PAGE: id=" + id + ", title=" + pageTitle + ",
uri=" + uri + ", status=" + status + ", comment="
- + comment
- + "]";
- }
- }
-
- static class Term {
- /** MD5 of the term */
- String md5;
- /** Term */
- String word;
-
- public Term(String word) {
- this.word = word;
- md5 = MD5(word);
- }
-
- public Term() {
- }
- }
-
- static class TermPosition {
- /** Term */
- String word;
- /** Page id */
- long pageId;
- /** Position List */
- int[] positions;
-
- public TermPosition() {
- }
- }
-
/** Document ID of fetching documents */
protected Map<Page, ClientGetter> runningFetch =
Collections.synchronizedMap(new HashMap<Page, ClientGetter>());
@@ -431,7 +345,7 @@
}
}
- protected class CallbackPrioritizer implements Comparator<Runnable> {
+ protected static class CallbackPrioritizer implements
Comparator<Runnable> {
public int compare(Runnable o1, Runnable o2) {
if (o1.getClass() == o2.getClass())
return 0;
@@ -920,40 +834,6 @@
Logger.minor(this, "Spider: indexes regenerated.");
}
- private static String convertToHex(byte[] data) {
- StringBuilder buf = new StringBuilder();
- for (int i = 0; i < data.length; i++) {
- int halfbyte = (data[i] >>> 4) & 0x0F;
- int two_halfs = 0;
- do {
- if ((0 <= halfbyte) && (halfbyte <= 9))
- buf.append((char) ('0' + halfbyte));
- else
- buf.append((char) ('a' + (halfbyte -
10)));
- halfbyte = data[i] & 0x0F;
- } while(two_halfs++ < 1);
- }
- return buf.toString();
- }
-
- /*
- * calculate the md5 for a given string
- */
- private static String MD5(String text) {
- try {
- MessageDigest md = MessageDigest.getInstance("MD5");
- byte[] md5hash = new byte[32];
- byte[] b = text.getBytes("UTF-8");
- md.update(b, 0, b.length);
- md5hash = md.digest();
- return convertToHex(md5hash);
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException("UTF-8 not supported", e);
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException("MD5 not supported", e);
- }
- }
-
public void generateSubIndex(String filename){
// generates the new subIndex
File outputFile = new File(filename);
_______________________________________________
cvs mailing list
[email protected]
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
