On Monday 15 December 2008 14:27, [email protected] wrote: > Author: j16sdiz > Date: 2008-12-15 14:27:46 +0000 (Mon, 15 Dec 2008) > New Revision: 24356 > > Modified: > trunk/plugins/XMLSpider/XMLSpider.java > Log: > use PageMaker for html > > Modified: trunk/plugins/XMLSpider/XMLSpider.java > =================================================================== > --- trunk/plugins/XMLSpider/XMLSpider.java 2008-12-15 13:25:09 UTC (rev 24355) > +++ trunk/plugins/XMLSpider/XMLSpider.java 2008-12-15 14:27:46 UTC (rev 24356) > @@ -15,6 +15,7 @@ > import java.security.MessageDigest; > import java.security.NoSuchAlgorithmException; > import java.util.ArrayList; > +import java.util.Collections; > import java.util.HashMap; > import java.util.HashSet; > import java.util.Iterator; > @@ -54,6 +55,7 @@ > import freenet.client.async.ClientCallback; > import freenet.client.async.ClientGetter; > import freenet.client.async.USKCallback; > +import freenet.clients.http.PageMaker; > import freenet.clients.http.filter.ContentFilter; > import freenet.clients.http.filter.FoundURICallback; > import freenet.clients.http.filter.UnsafeContentTypeException; > @@ -69,7 +71,7 @@ > import freenet.pluginmanager.FredPluginVersioned; > import freenet.pluginmanager.PluginHTTPException; > import freenet.pluginmanager.PluginRespirator; > -import freenet.support.HTMLEncoder; > +import freenet.support.HTMLNode; > import freenet.support.Logger; > import freenet.support.api.Bucket; > import freenet.support.api.HTTPRequest; > @@ -160,7 +162,7 @@ > } > > /** Document ID of fetching documents */ > - protected Map<Page, ClientGetter> runningFetch = new HashMap<Page, ClientGetter>(); > + protected Map<Page, ClientGetter> runningFetch = Collections.synchronizedMap(new HashMap<Page, ClientGetter>()); > > long tProducedIndex; > protected AtomicLong maxPageId; > @@ -210,7 +212,8 @@ > // Any lower makes it very difficult to debug. Maybe reduce for production - after solving the ARK bugs. > private final short PRIORITY_CLASS = RequestStarter.IMMEDIATE_SPLITFILE_PRIORITY_CLASS; > private boolean stopped = true; > - PluginRespirator pr; > + > + private PageMaker pageMaker; > > private final static String[] BADLIST_EXTENSTION = new String[] { > ".ico", ".bmp", ".png", ".jpg", ".gif", // image > @@ -971,11 +974,14 @@ > } > > public void runPlugin(PluginRespirator pr){ > - this.pr = pr; > this.core = pr.getNode().clientCore; > > + this.pageMaker = pr.getPageMaker(); > + pageMaker.addNavigationLink("/plugins/plugins.XMLSpider.XMLSpider", "Home", "Home page", false, null); > + pageMaker.addNavigationLink("/plugins/", "Plugins page", "Back > to Plugins page", false, null); > + > /* Initialize Fetch Context */ > - this.ctx = core.makeClient((short) 0).getFetchContext(); > + this.ctx = pr.getHLSimpleClient().getFetchContext(); > ctx.maxSplitfileBlockRetries = 2; > ctx.maxNonSplitfileRetries = 2; > ctx.maxTempLength = 2 * 1024 * 1024; > @@ -1018,192 +1024,85 @@ > } > }, "Spider Plugin Starter"); > } > + > + private long getPageCount(Status status) { > + Query query = db.query(); > + query.constrain(Page.class); > + query.descend("status").constrain(status); > + ObjectSet<Page> set = query.execute(); > > - /** > - * Interface to the Spider data > - */ > - public String handleHTTPGet(HTTPRequest request) throws PluginHTTPException{ > - StringBuilder out = new StringBuilder(); > + return set.size(); > + } > + > + private void listPage(Status status, HTMLNode parent) { > + Query query = db.query(); > + query.constrain(Page.class); > + query.descend("status").constrain(status); > + query.descend("lastChange").orderDescending(); > + ObjectSet<Page> set = query.execute(); > > - String listname = request.getParam("list"); > - if(listname.length() != 0) > - { > - appendDefaultHeader(out,null); > - out.append("<p><h4>"+listname+" URIs</h4></p>"); > - appendList(listname, out); > - return out.toString(); > - } > - appendDefaultPageStart(out,null); > - String uriParam = request.getParam("adduri"); > - if(uriParam != null && uriParam.length() != 0) > - { > - try { > - FreenetURI uri = new FreenetURI(uriParam); > - synchronized (this) { > - // Check if queued already > - Page page = getPageByURI(uri); > - if (page != null) { > - // We have no reliable way to > stop a request, > - // requeue only if it is > successed / failed > - if (page.status == > Status.SUCCEEDED || page.status == Status.FAILED) { > - page.lastChange = > System.currentTimeMillis(); > - page.status = > Status.QUEUED; > + HTMLNode list = parent.addChild("ol", "style", "overflow: > scroll"); > > - db.store(page); > - } > - } > - } > - out.append("<p>URI added :"+uriParam+"</p>"); > - queueURI(uri, "manually"); > - startSomeRequests(); > - } catch (MalformedURLException mue1) { > - out.append("<p>MalFormed URI: "+uriParam+"</p"); > - } > + for (int i = 0; i < maxShownURIs && set.hasNext(); i++) { > + Page page = set.next(); > + list.addChild("li", new String[] { "title" }, new > String[] { page.comment }, page.uri); > } > - return out.toString(); > } > -/* > - * List the visited, queued, failed and running fetches on the web interface > - */ > - private synchronized void appendList(String listname, StringBuilder > out) { > - Iterable<Page> it = runningFetch.keySet(); > > - if (listname.equals("running")) { > - it = runningFetch.keySet(); > - } else if (listname.equals("visited")) { > - Query query = db.query(); > - query.constrain(Page.class); > - query.descend("status").constrain(Status.SUCCEEDED); > - query.descend("lastChange").orderAscending(); > - ObjectSet<Page> set = query.execute(); > + /** > + * Interface to the Spider data > + */ > + public String handleHTTPGet(HTTPRequest request) throws PluginHTTPException{ > + HTMLNode pageNode = pageMaker.getPageNode(pluginName, null); > + HTMLNode contentNode = pageMaker.getContentNode(pageNode); > > - it = set; > - } else if (listname.equals("queued")) { > - Query query = db.query(); > - query.constrain(Page.class); > - query.descend("status").constrain(Status.QUEUED); > - query.descend("lastChange").orderAscending(); > - ObjectSet<Page> set = query.execute(); > + HTMLNode overviewTable = contentNode.addChild("table", "class", "column"); > + HTMLNode overviewTableRow = overviewTable.addChild("tr"); > > - it = set; > - } else if (listname.equals("failed")) { > - Query query = db.query(); > - query.constrain(Page.class); > - query.descend("status").constrain(Status.FAILED); > - query.descend("lastChange").orderAscending(); > - ObjectSet<Page> set = query.execute(); > + // Column 1 > + HTMLNode nextTableCell = overviewTableRow.addChild("td", "class", "first"); > + HTMLNode statusBox = pageMaker.getInfobox("Spider Status"); > + statusBox.addChild("%", "Running Request: " + > runningFetch.size() + "/" + maxParallelRequests); > + statusBox.addChild("br"); > + statusBox.addChild("%", "Queued: " + > getPageCount(Status.QUEUED)); > + statusBox.addChild("br"); > + statusBox.addChild("%", "Succeeded: " + > getPageCount(Status.SUCCEEDED)); > + statusBox.addChild("br"); > + statusBox.addChild("%", "Failed: " + > getPageCount(Status.FAILED));
Please don't use % unless you actually need to. Use # for plain text.
> + nextTableCell.addChild(statusBox);
>
> - it = set;
> - }
> + // Column 2
> + nextTableCell = overviewTableRow.addChild("td", "class",
> "second");
> + statusBox = pageMaker.getInfobox("Spider Status");
> + nextTableCell.addChild(statusBox);
>
> - for (Page page : it)
> - out.append("<code title=\"" +
> HTMLEncoder.encode(page.comment) + "\">" +
HTMLEncoder.encode(page.uri)
> - + "</code><br/>");
> - }
> + HTMLNode runningList = pageMaker.getInfobox("Running URI");
> + synchronized (runningFetch) {
> + HTMLNode list = runningList.addChild("ol", "style",
> "overflow: scroll");
>
> - private void appendDefaultPageStart(StringBuilder out, String
> stylesheet)
{
> -
> - out.append("<HTML><HEAD><TITLE>" + pluginName + "</TITLE>");
> - if(stylesheet != null)
> - out.append("<link href=\""+stylesheet+"\"
> type=\"text/css\"
rel=\"stylesheet\" />");
> - out.append("</HEAD><BODY>\n");
> - out.append("<CENTER><H1>" + pluginName +
> "</H1><BR/><BR/><BR/>\n");
> - out.append("Add uri:");
> - out.append("<form method=\"GET\"><input type=\"text\"
name=\"adduri\" /><br/><br/>");
> - out.append("<input type=\"submit\" value=\"Add uri\"
> /></form>");
> - List<Page> runningFetchesSnapshot;
> - long runningFetchesSnapshotSize;
> - List<Page> visitedSnapshot;
> - long visitedSnapshotSize;
> - List<Page> failedSnapshot;
> - long failedSnapshotSize;
> - List<Page> queuedSnapshot;
> - long queuedSnapshotSize;
> -
> - synchronized(this) {
> - runningFetchesSnapshot = new
> ArrayList<Page>(maxShownURIs);
> - {
> - Iterator<Page> it =
> this.runningFetch.keySet().iterator();
> - for (int i = 0; it.hasNext() && i <
> maxShownURIs; i++)
> - runningFetchesSnapshot.add(it.next());
> - runningFetchesSnapshotSize =
> runningFetch.size();
> + Iterator<Page> pi = runningFetch.keySet().iterator();
> + for (int i = 0; i < maxShownURIs && pi.hasNext(); i++) {
> + Page page = pi.next();
> + list.addChild("li", new String[] { "title" },
> new String[] {
page.comment }, page.uri);
> }
> -
> - visitedSnapshot = new ArrayList<Page>(maxShownURIs);
> - Query query = db.query();
> - query.constrain(Page.class);
> - query.descend("status").constrain(Status.SUCCEEDED);
> - query.descend("lastChange").orderAscending();
> - ObjectSet<Page> set = query.execute();
> - for (int i = 0; set.hasNext() && i < maxShownURIs; i++)
> - visitedSnapshot.add(set.next());
> - visitedSnapshotSize = set.size();
> -
> - failedSnapshot = new ArrayList<Page>(maxShownURIs);
> - query = db.query();
> - query.constrain(Page.class);
> - query.descend("status").constrain(Status.FAILED);
> - query.descend("lastChange").orderAscending();
> - set = query.execute();
> - for (int i = 0; set.hasNext() && i < maxShownURIs; i++)
> - failedSnapshot.add(set.next());
> - failedSnapshotSize = set.size();
> -
> - queuedSnapshot = new ArrayList<Page>(maxShownURIs);
> - query = db.query();
> - query.constrain(Page.class);
> - query.descend("status").constrain(Status.QUEUED);
> - query.descend("lastChange").orderAscending();
> - set = query.execute();
> - for (int i = 0; set.hasNext() && i < maxShownURIs; i++)
> - queuedSnapshot.add(set.next());
> - queuedSnapshotSize = set.size();
> }
> -
> - out.append("<p><h3>Running Fetches</h3></p>");
> - out.append("<br/>Size :" + runningFetchesSnapshotSize +
> "<br/>");
> - for (Page page : runningFetchesSnapshot)
> - out.append("<code title=\"" +
> HTMLEncoder.encode(page.comment) + "\">" +
HTMLEncoder.encode(page.uri)
> - + "</code><br/>");
> - out.append("<p><a href=\"?list="+"running"+"\">Show
> all</a><br/></p>");
> + contentNode.addChild(runningList);
>
> -
> - out.append("<p><h3>Queued URIs</h3></p>");
> - out.append("<br/>Size :" + queuedSnapshotSize + "<br/>");
> - for (Page page : queuedSnapshot)
> - out.append("<code title=\"" +
> HTMLEncoder.encode(page.comment) + "\">" +
HTMLEncoder.encode(page.uri)
> - + "</code><br/>");
> - out.append("<p><a href=\"?list=\">Show all</a><br/></p>");
> -
> -
> - out.append("<p><h3>Visited URIs</h3></p>");
> - out.append("<br/>Size :" + visitedSnapshotSize + "<br/>");
> - for (Page page : visitedSnapshot)
> - out.append("<code title=\"" +
> HTMLEncoder.encode(page.comment) + "\">" +
HTMLEncoder.encode(page.uri)
> - + "</code><br/>");
> - out.append("<p><a href=\"?list="+"visited"+"\">Show
> all</a><br/></p>");
> -
> - out.append("<p><h3>Failed URIs</h3></p>");
> - out.append("<br/>Size :" + failedSnapshotSize + "<br/>");
> - for (Page page : failedSnapshot)
> - out.append("<code title=\"" +
> HTMLEncoder.encode(page.comment) + "\">" +
HTMLEncoder.encode(page.uri)
> - + "</code><br/>");
> - out.append("<p><a href=\"?list="+"failed"+"\">Show
> all</a><br/></p>");
> - out.append("<p>Time taken in generating index =
> "+time_taken+"</p>");
> - }
> + HTMLNode queuedList = pageMaker.getInfobox("Queued URI");
> + listPage(Status.QUEUED, queuedList);
> + contentNode.addChild(queuedList);
>
> + HTMLNode succeededList = pageMaker.getInfobox("Succeeded URI");
> + listPage(Status.SUCCEEDED, succeededList);
> + contentNode.addChild(succeededList);
>
> - private void appendDefaultHeader(StringBuilder out, String stylesheet){
> - out.append("<HTML><HEAD><TITLE>" + pluginName + "</TITLE>");
> - if(stylesheet != null)
> - out.append("<link href=\""+stylesheet+"\"
> type=\"text/css\"
rel=\"stylesheet\" />");
> - out.append("</HEAD><BODY>\n");
> - out.append("<CENTER><H1>" + pluginName +
> "</H1><BR/><BR/><BR/>\n");
> - out.append("Add uri:");
> - out.append("<form method=\"GET\"><input type=\"text\"
name=\"adduri\" /><br/><br/>");
> - out.append("<input type=\"submit\" value=\"Add uri\"
> /></form>");
> - }
> + HTMLNode failedList = pageMaker.getInfobox("Failed URI");
> + listPage(Status.FAILED, failedList);
> + contentNode.addChild(failedList);
>
> + return pageNode.generate();
> + }
> +
> /**
> * creates the callback object for each page.
> *<p>Used to create inlinks and outlinks for each page separately.
>
> _______________________________________________
> cvs mailing list
> [email protected]
> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
>
>
pgpmyFJ9ZFogW.pgp
Description: PGP signature
_______________________________________________ Devl mailing list [email protected] http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl
