Author: toad
Date: 2008-03-05 19:57:13 +0000 (Wed, 05 Mar 2008)
New Revision: 18377
Modified:
trunk/freenet/src/freenet/client/HighLevelSimpleClient.java
trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
trunk/freenet/src/freenet/clients/http/FProxyToadlet.java
trunk/freenet/src/freenet/clients/http/NinjaSpider.java
trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java
trunk/freenet/src/freenet/clients/http/Spider.java
trunk/freenet/src/freenet/clients/http/filter/CSSParser.java
trunk/freenet/src/freenet/clients/http/filter/FilterCallback.java
trunk/freenet/src/freenet/clients/http/filter/FoundURICallback.java
trunk/freenet/src/freenet/clients/http/filter/GenericReadFilterCallback.java
trunk/freenet/src/freenet/clients/http/filter/HTMLFilter.java
trunk/freenet/src/freenet/clients/http/filter/NullFilterCallback.java
trunk/freenet/src/freenet/node/simulator/RealNodeULPRTest.java
trunk/plugins/XMLSpider/XMLSpider.java
Log:
Inline prefetch. Off by default.
I had hoped this would significantly improve performance with browsers that
only allow a few connections.
However, testing suggests that this is not the case. Probably the bottleneck is
a combination of bad peers and the node logic not wanting to do many local
requests at the expense of remote ones (for anonymity).
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClient.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClient.java 2008-03-05
19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClient.java 2008-03-05
19:57:13 UTC (rev 18377)
@@ -4,6 +4,7 @@
package freenet.client;
import java.util.HashMap;
+import java.util.Set;
import freenet.client.events.ClientEventListener;
import freenet.keys.FreenetURI;
@@ -77,4 +78,12 @@
*/
public FreenetURI[] generateKeyPair(String docName);
+ /**
+ * Prefetch a key at a very low priority. If it hasn't been fetched
within the timeout,
+ * kill the fetch.
+ * @param uri
+ * @param timeout
+ */
+ public void prefetch(FreenetURI uri, long timeout, long maxSize, Set
allowedTypes);
+
}
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2008-03-05 19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2008-03-05 19:57:13 UTC (rev 18377)
@@ -5,8 +5,11 @@
import java.io.IOException;
import java.util.HashMap;
+import java.util.Set;
import freenet.client.async.BackgroundBlockEncoder;
+import freenet.client.async.BaseClientPutter;
+import freenet.client.async.ClientCallback;
import freenet.client.async.ClientGetter;
import freenet.client.async.ClientPutter;
import freenet.client.async.HealingQueue;
@@ -20,10 +23,12 @@
import freenet.keys.InsertableClientSSK;
import freenet.node.NodeClientCore;
import freenet.node.RequestScheduler;
+import freenet.node.RequestStarter;
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.api.BucketFactory;
import freenet.support.io.BucketTools;
+import freenet.support.io.NullBucket;
import freenet.support.io.NullPersistentFileTracker;
import freenet.support.io.PersistentFileTracker;
@@ -208,4 +213,54 @@
return new FreenetURI[] { key.getInsertURI(), key.getURI() };
}
+ private final ClientCallback nullCallback = new ClientCallback() {
+
+ public void onFailure(FetchException e, ClientGetter state) {
+ // Ignore
+ }
+
+ public void onFailure(InsertException e, BaseClientPutter
state) {
+ // Impossible
+ }
+
+ public void onFetchable(BaseClientPutter state) {
+ // Impossible
+ }
+
+ public void onGeneratedURI(FreenetURI uri, BaseClientPutter
state) {
+ // Impossible
+ }
+
+ public void onMajorProgress() {
+ // Ignore
+ }
+
+ public void onSuccess(FetchResult result, ClientGetter state) {
+ result.data.free();
+ }
+
+ public void onSuccess(BaseClientPutter state) {
+ // Impossible
+ }
+
+ };
+
+ public void prefetch(FreenetURI uri, long timeout, long maxSize, Set
allowedTypes) {
+ FetchContext ctx = getFetchContext(maxSize);
+ ctx.allowedMIMETypes = allowedTypes;
+ final ClientGetter get = new ClientGetter(nullCallback,
core.requestStarters.chkFetchScheduler, core.requestStarters.sskFetchScheduler,
uri, ctx, RequestStarter.PREFETCH_PRIORITY_CLASS, this, new NullBucket(), null);
+ ctx.ticker.queueTimedJob(new Runnable() {
+
+ public void run() {
+ get.cancel();
+ }
+
+ }, timeout);
+ try {
+ get.start();
+ } catch (FetchException e) {
+ // Ignore
+ }
+ }
+
}
Modified: trunk/freenet/src/freenet/clients/http/FProxyToadlet.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/FProxyToadlet.java 2008-03-05
19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/clients/http/FProxyToadlet.java 2008-03-05
19:57:13 UTC (rev 18377)
@@ -7,6 +7,8 @@
import java.net.SocketException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.HashSet;
+import java.util.Set;
import freenet.client.DefaultMIMETypes;
import freenet.client.FetchException;
@@ -14,6 +16,7 @@
import freenet.client.HighLevelSimpleClient;
import freenet.clients.http.bookmark.BookmarkManager;
import freenet.clients.http.filter.ContentFilter;
+import freenet.clients.http.filter.FoundURICallback;
import freenet.clients.http.filter.UnsafeContentTypeException;
import freenet.clients.http.filter.ContentFilter.FilterOutput;
import freenet.config.Config;
@@ -40,6 +43,16 @@
private static byte[] random;
final NodeClientCore core;
+ private static FoundURICallback prefetchHook;
+ private static boolean ENABLE_PREFETCH = true;
+ static final Set prefetchAllowedTypes = new HashSet();
+ static {
+ // Only valid inlines
+ prefetchAllowedTypes.add("image/png");
+ prefetchAllowedTypes.add("image/jpeg");
+ prefetchAllowedTypes.add("image/gif");
+ }
+
// ?force= links become invalid after 2 hours.
private static final long FORCE_GRAIN_INTERVAL = 60*60*1000;
/** Maximum size for transparent pass-through, should be a config
option */
@@ -54,11 +67,30 @@
}
}
- public FProxyToadlet(HighLevelSimpleClient client, NodeClientCore core)
{
+ public FProxyToadlet(final HighLevelSimpleClient client, NodeClientCore
core) {
super(client);
client.setMaxLength(MAX_LENGTH);
client.setMaxIntermediateLength(MAX_LENGTH);
this.core = core;
+ if(ENABLE_PREFETCH) {
+ prefetchHook = new FoundURICallback() {
+
+ public void foundURI(FreenetURI uri) {
+ // Ignore
+ }
+
+ public void foundURI(FreenetURI uri, boolean
inline) {
+ if(!inline) return;
+ if(Logger.shouldLog(Logger.MINOR,
this)) Logger.minor(this, "Prefetching "+uri);
+ client.prefetch(uri, 60*1000, 512*1024,
prefetchAllowedTypes);
+ }
+
+ public void onText(String text, String type,
URI baseURI) {
+ // Ignore
+ }
+
+ };
+ }
}
public String supportedMethods() {
@@ -103,7 +135,7 @@
try {
if((!force) && (!forceDownload)) {
- FilterOutput fo = ContentFilter.filter(data,
bucketFactory, mimeType, key.toURI(basePath), null);
+ FilterOutput fo = ContentFilter.filter(data,
bucketFactory, mimeType, key.toURI(basePath), prefetchHook);
data = fo.data;
mimeType = fo.type;
Modified: trunk/freenet/src/freenet/clients/http/NinjaSpider.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/NinjaSpider.java 2008-03-05
19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/clients/http/NinjaSpider.java 2008-03-05
19:57:13 UTC (rev 18377)
@@ -235,6 +235,11 @@
startSomeRequests();
}
+ public void foundURI(FreenetURI uri, boolean inline) {
+ queueURI(uri);
+ startSomeRequests();
+ }
+
public void onText(String s, String type, URI baseURI) {
FreenetURI uri;
Modified: trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java
2008-03-05 19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java
2008-03-05 19:57:13 UTC (rev 18377)
@@ -74,6 +74,7 @@
private boolean doRobots;
public BookmarkManager bookmarkManager;
private boolean enablePersistentConnections;
+ private boolean enableInlinePrefetch;
static boolean isPanicButtonToBeShown;
public static final int DEFAULT_FPROXY_PORT = 8888;
@@ -369,7 +370,30 @@
}
});
enablePersistentConnections =
fproxyConfig.getBoolean("enablePersistentConnections");
-
+
+ // Off by default.
+ // I had hoped it would yield a significant performance boost
to bootstrap performance
+ // on browsers with low numbers of simultaneous connections.
Unfortunately the bottleneck
+ // appears to be that the node does very few local requests
compared to external requests
+ // (for anonymity's sake).
+
+ fproxyConfig.register("enableInlinePrefetch", false,
configItemOrder++, false, false, "SimpleToadletServer.enableInlinePrefetch",
"SimpleToadletServer.enableInlinePrefetchLong",
+ new BooleanCallback() {
+
+ public boolean get() {
+
synchronized(SimpleToadletServer.this) {
+ return
enableInlinePrefetch;
+ }
+ }
+
+ public void set(boolean val) throws
InvalidConfigValueException {
+
synchronized(SimpleToadletServer.this) {
+ enableInlinePrefetch =
val;
+ }
+ }
+ });
+ enableInlinePrefetch =
fproxyConfig.getBoolean("enableInlinePrefetch");
+
fproxyConfig.register("allowedHosts",
"127.0.0.1,0:0:0:0:0:0:0:1", configItemOrder++, true, true,
"SimpleToadletServer.allowedHosts", "SimpleToadletServer.allowedHostsLong",
new FProxyAllowedHostsCallback());
fproxyConfig.register("allowedHostsFullAccess",
"127.0.0.1,0:0:0:0:0:0:0:1", configItemOrder++, true, true,
"SimpleToadletServer.allowedFullAccess",
Modified: trunk/freenet/src/freenet/clients/http/Spider.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/Spider.java 2008-03-05 19:49:02 UTC
(rev 18376)
+++ trunk/freenet/src/freenet/clients/http/Spider.java 2008-03-05 19:57:13 UTC
(rev 18377)
@@ -178,6 +178,11 @@
queueURI(uri);
startSomeRequests();
}
+
+ public void foundURI(FreenetURI uri, boolean inline) {
+ queueURI(uri);
+ startSomeRequests();
+ }
public void onText(String s, String type, URI baseURI) {
Modified: trunk/freenet/src/freenet/clients/http/filter/CSSParser.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/filter/CSSParser.java
2008-03-05 19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/clients/http/filter/CSSParser.java
2008-03-05 19:57:13 UTC (rev 18377)
@@ -42,11 +42,11 @@
}
String processImportURL(String s) throws CommentException {
- return HTMLFilter.sanitizeURI(HTMLFilter.stripQuotes(s),
"text/css", null, cb);
+ return HTMLFilter.sanitizeURI(HTMLFilter.stripQuotes(s),
"text/css", null, cb, true);
}
String processURL(String s) throws CommentException {
- return HTMLFilter.sanitizeURI(HTMLFilter.stripQuotes(s), null,
null, cb);
+ return HTMLFilter.sanitizeURI(HTMLFilter.stripQuotes(s), null,
null, cb, true);
}
void log(String s) {
Modified: trunk/freenet/src/freenet/clients/http/filter/FilterCallback.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/filter/FilterCallback.java
2008-03-05 19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/clients/http/filter/FilterCallback.java
2008-03-05 19:57:13 UTC (rev 18377)
@@ -15,6 +15,14 @@
public String processURI(String uri, String overrideType) throws
CommentException;
/**
+ * Process a URI.
+ * If it cannot be turned into something sufficiently safe, then return
null.
+ * @param overrideType Force the return type.
+ * @throws CommentException If the URI is nvalid or unacceptable in
some way.
+ */
+ public String processURI(String uri, String overrideType, boolean
noRelative, boolean inline) throws CommentException;
+
+ /**
* Process a base URI in the page. Not only is this filtered, it
affects all
* relative uri's on the page.
*/
Modified: trunk/freenet/src/freenet/clients/http/filter/FoundURICallback.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/filter/FoundURICallback.java
2008-03-05 19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/clients/http/filter/FoundURICallback.java
2008-03-05 19:57:13 UTC (rev 18377)
@@ -17,6 +17,13 @@
public void foundURI(FreenetURI uri);
/**
+ * Called when a Freenet URI is found.
+ * @param uri The URI.
+ * FIXME: Indicate the type of the link e.g. inline image, hyperlink,
etc??
+ */
+ public void foundURI(FreenetURI uri, boolean inline);
+
+ /**
* Called when some plain text is processed. This is used typically by
* spiders to index pages by their content.
* @param text The text. Will already have been fed through whatever
decoding
Modified:
trunk/freenet/src/freenet/clients/http/filter/GenericReadFilterCallback.java
===================================================================
---
trunk/freenet/src/freenet/clients/http/filter/GenericReadFilterCallback.java
2008-03-05 19:49:02 UTC (rev 18376)
+++
trunk/freenet/src/freenet/clients/http/filter/GenericReadFilterCallback.java
2008-03-05 19:57:13 UTC (rev 18377)
@@ -54,10 +54,10 @@
}
public String processURI(String u, String overrideType) throws
CommentException {
- return processURI(u, overrideType, false);
+ return processURI(u, overrideType, false, false);
}
- public String processURI(String u, String overrideType, boolean
noRelative) throws CommentException {
+ public String processURI(String u, String overrideType, boolean
noRelative, boolean inline) throws CommentException {
URI uri;
URI resolved;
boolean logMINOR = Logger.shouldLog(Logger.MINOR, this);
@@ -113,7 +113,7 @@
}
FreenetURI furi = new FreenetURI(p);
if(logMINOR) Logger.minor(this, "Parsed:
"+furi);
- return processURI(furi, uri, overrideType,
noRelative);
+ return processURI(furi, uri, overrideType,
noRelative, inline);
} catch (MalformedURLException e) {
// Not a FreenetURI
if(logMINOR) Logger.minor(this, "Malformed URL
(a): "+e, e);
@@ -139,7 +139,7 @@
while(p.startsWith("/")) p = p.substring(1);
FreenetURI furi = new FreenetURI(p);
if(logMINOR) Logger.minor(this, "Parsed:
"+furi);
- return processURI(furi, uri, overrideType,
noRelative);
+ return processURI(furi, uri, overrideType,
noRelative, inline);
} catch (MalformedURLException e) {
if(logMINOR) Logger.minor(this, "Malformed URL
(b): "+e, e);
if(e.getMessage() != null) {
@@ -201,18 +201,19 @@
}
}
- private String processURI(FreenetURI furi, URI uri, String
overrideType, boolean noRelative) {
+ private String processURI(FreenetURI furi, URI uri, String
overrideType, boolean noRelative, boolean inline) {
// Valid Freenet URI, allow it
// Now what about the queries?
HTTPRequest req = new HTTPRequestImpl(uri);
if(cb != null) cb.foundURI(furi);
+ if(cb != null) cb.foundURI(furi, inline);
return finishProcess(req, overrideType, '/' +
furi.toString(false, false), uri, noRelative);
}
public String onBaseHref(String baseHref) {
String ret;
try {
- ret = processURI(baseHref, null, true);
+ ret = processURI(baseHref, null, true, false);
} catch (CommentException e1) {
Logger.error(this, "Failed to parse base href:
"+baseHref+" -> "+e1.getMessage());
ret = null;
Modified: trunk/freenet/src/freenet/clients/http/filter/HTMLFilter.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/filter/HTMLFilter.java
2008-03-05 19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/clients/http/filter/HTMLFilter.java
2008-03-05 19:57:13 UTC (rev 18377)
@@ -584,7 +584,8 @@
new TagVerifier(
"head",
new String[] { "id" },
- new String[] { "profile" }));
+ new String[] { "profile" },
+ null));
allowedTagsVerifiers.put(
"title",
new TagVerifier("title", new String[] { "id" }));
@@ -594,6 +595,7 @@
new CoreTagVerifier(
"body",
new String[] { "bgcolor", "text", "link",
"vlink", "alink" },
+ null,
new String[] { "background" },
new String[] { "onload", "onunload" }));
String[] group =
@@ -605,6 +607,7 @@
group[x],
new String[] { "align" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
String[] group2 =
{
@@ -648,6 +651,7 @@
group2[x],
emptyStringArray,
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"blockquote",
@@ -655,6 +659,7 @@
"blockquote",
emptyStringArray,
new String[] { "cite" },
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"q",
@@ -662,12 +667,14 @@
"q",
emptyStringArray,
new String[] { "cite" },
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"br",
new BaseCoreTagVerifier(
"br",
new String[] { "clear" },
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"pre",
@@ -675,6 +682,7 @@
"pre",
new String[] { "width", "xml:space" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"ins",
@@ -682,6 +690,7 @@
"ins",
new String[] { "datetime" },
new String[] { "cite" },
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"del",
@@ -689,6 +698,7 @@
"del",
new String[] { "datetime" },
new String[] { "cite" },
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"ul",
@@ -696,6 +706,7 @@
"ul",
new String[] { "type", "compact" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"ol",
@@ -703,6 +714,7 @@
"ol",
new String[] { "type", "compact", "start" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"li",
@@ -710,6 +722,7 @@
"li",
new String[] { "type", "value" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"dl",
@@ -717,6 +730,7 @@
"dl",
new String[] { "compact" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"dir",
@@ -724,6 +738,7 @@
"dir",
new String[] { "compact" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"menu",
@@ -731,6 +746,7 @@
"menu",
new String[] { "compact" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"table",
@@ -746,6 +762,7 @@
"cellpadding",
"align",
"bgcolor" },
+ emptyStringArray,
new String[] { "background" },
emptyStringArray));
allowedTagsVerifiers.put(
@@ -754,6 +771,7 @@
"thead",
new String[] { "align", "char", "charoff",
"valign" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"tfoot",
@@ -761,6 +779,7 @@
"tfoot",
new String[] { "align", "char", "charoff",
"valign" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"tbody",
@@ -768,6 +787,7 @@
"tbody",
new String[] { "align", "char", "charoff",
"valign" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"colgroup",
@@ -781,6 +801,7 @@
"charoff",
"valign" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"col",
@@ -794,6 +815,7 @@
"charoff",
"valign" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"tr",
@@ -806,6 +828,7 @@
"valign",
"bgcolor" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"th",
@@ -826,6 +849,7 @@
"bgcolor",
"width",
"height" },
+ emptyStringArray,
new String[] { "background" },
emptyStringArray));
allowedTagsVerifiers.put(
@@ -847,6 +871,7 @@
"bgcolor",
"width",
"height" },
+ emptyStringArray,
new String[] { "background" },
emptyStringArray));
allowedTagsVerifiers.put(
@@ -861,6 +886,7 @@
"coords",
"target" },
emptyStringArray,
+ emptyStringArray,
new String[] { "onfocus", "onblur" }));
allowedTagsVerifiers.put(
"link",
@@ -868,6 +894,7 @@
"link",
new String[] { "media", "target" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"base",
@@ -889,7 +916,8 @@
"border",
"hspace",
"vspace" },
- new String[] { "src", "longdesc", "usemap" },
+ new String[] { "longdesc", "usemap" },
+ new String[] { "src" },
emptyStringArray));
// FIXME: object tag -
// http://www.w3.org/TR/html4/struct/objects.html#h-13.3
@@ -902,6 +930,7 @@
"map",
new String[] { "name" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"area",
@@ -916,6 +945,7 @@
"alt",
"target" },
new String[] { "href" },
+ emptyStringArray,
new String[] { "onfocus", "onblur" }));
allowedTagsVerifiers.put("style", new StyleTagVerifier());
allowedTagsVerifiers.put(
@@ -923,12 +953,14 @@
new BaseCoreTagVerifier(
"font",
new String[] { "size", "color", "face" },
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"basefont",
new BaseCoreTagVerifier(
"basefont",
new String[] { "size", "color", "face" },
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"hr",
@@ -936,6 +968,7 @@
"hr",
new String[] { "align", "noshade", "size",
"width" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"frameset",
@@ -943,6 +976,7 @@
"frameset",
new String[] { "rows", "cols" },
emptyStringArray,
+ emptyStringArray,
new String[] { "onload", "onunload" },
false));
allowedTagsVerifiers.put(
@@ -956,7 +990,8 @@
"marginheight",
"noresize",
"scrolling" },
- new String[] { "longdesc", "src" }));
+ new String[] { "longdesc" },
+ new String[] { "src" }));
allowedTagsVerifiers.put(
"iframe",
new BaseCoreTagVerifier(
@@ -970,7 +1005,8 @@
"align",
"height",
"width" },
- new String[] { "longdesc", "src" }));
+ new String[] { "longdesc"},
+ new String[] { "src" }));
allowedTagsVerifiers.put(
"form",
@@ -1000,7 +1036,8 @@
"ismap",
"accept",
"align" },
- new String[] { "src", "usemap" },
+ new String[] { "usemap" },
+ new String[] { "src" },
new String[] { "onfocus", "onblur", "onselect",
"onchange" }));
allowedTagsVerifiers.put(
"button",
@@ -1014,6 +1051,7 @@
"type",
"disabled" },
emptyStringArray,
+ emptyStringArray,
new String[] { "onfocus", "onblur" }));
allowedTagsVerifiers.put(
"select",
@@ -1026,6 +1064,7 @@
"disabled",
"tabindex" },
emptyStringArray,
+ emptyStringArray,
new String[] { "onfocus", "onblur", "onchange"
}));
allowedTagsVerifiers.put(
"optgroup",
@@ -1033,6 +1072,7 @@
"optgroup",
new String[] { "disabled", "label" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"option",
@@ -1040,6 +1080,7 @@
"option",
new String[] { "selected", "disabled", "label",
"value" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"textarea",
@@ -1054,12 +1095,14 @@
"disabled",
"readonly" },
emptyStringArray,
+ emptyStringArray,
new String[] { "onfocus", "onblur", "onselect",
"onchange" }));
allowedTagsVerifiers.put(
"isindex",
new BaseCoreTagVerifier(
"isindex",
new String[] { "prompt" },
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put(
"label",
@@ -1067,6 +1110,7 @@
"label",
new String[] { "for", "accesskey" },
emptyStringArray,
+ emptyStringArray,
new String[] { "onfocus", "onblur" }));
allowedTagsVerifiers.put(
"legend",
@@ -1074,6 +1118,7 @@
"legend",
new String[] { "accesskey", "align" },
emptyStringArray,
+ emptyStringArray,
emptyStringArray));
allowedTagsVerifiers.put("script", new ScriptTagVerifier());
}
@@ -1082,12 +1127,13 @@
final String tag;
final HashSet allowedAttrs;
final HashSet uriAttrs;
+ final HashSet inlineURIAttrs;
TagVerifier(String tag, String[] allowedAttrs) {
- this(tag, allowedAttrs, null);
+ this(tag, allowedAttrs, null, null);
}
- TagVerifier(String tag, String[] allowedAttrs, String[]
uriAttrs) {
+ TagVerifier(String tag, String[] allowedAttrs, String[]
uriAttrs, String[] inlineURIAttrs) {
this.tag = tag;
this.allowedAttrs = new HashSet();
if (allowedAttrs != null) {
@@ -1099,6 +1145,11 @@
for (int x = 0; x < uriAttrs.length; x++)
this.uriAttrs.add(uriAttrs[x]);
}
+ this.inlineURIAttrs = new HashSet();
+ if (inlineURIAttrs != null) {
+ for (int x = 0; x < inlineURIAttrs.length; x++)
+
this.inlineURIAttrs.add(inlineURIAttrs[x]);
+ }
}
ParsedTag sanitize(ParsedTag t, HTMLParseContext pc) throws
DataFilterException {
@@ -1182,12 +1233,13 @@
continue;
}
if (uriAttrs.contains(x)) {
+ if(logMINOR) Logger.minor(this,
"Non-inline URI attribute: "+x);
// URI
if (o instanceof String) {
// Java's URL handling doesn't
seem suitable
String uri = (String) o;
uri = HTMLDecoder.decode(uri);
- uri = htmlSanitizeURI(uri,
null, null, pc.cb, pc);
+ uri = htmlSanitizeURI(uri,
null, null, pc.cb, pc, false);
if (uri != null) {
uri =
HTMLEncoder.encode(uri);
hn.put(x, uri);
@@ -1195,6 +1247,21 @@
}
// FIXME: rewrite absolute URLs, handle
?date= etc
}
+ if (inlineURIAttrs.contains(x)) {
+ if(logMINOR) Logger.minor(this, "Inline
URI attribute: "+x);
+ // URI
+ if (o instanceof String) {
+ // Java's URL handling doesn't
seem suitable
+ String uri = (String) o;
+ uri = HTMLDecoder.decode(uri);
+ uri = htmlSanitizeURI(uri,
null, null, pc.cb, pc, true);
+ if (uri != null) {
+ uri =
HTMLEncoder.encode(uri);
+ hn.put(x, uri);
+ }
+ }
+ // FIXME: rewrite absolute URLs, handle
?date= etc
+ }
}
// lang, xml:lang and dir can go on anything
// lang or xml:lang = language [ "-" country [ "-"
variant ] ]
@@ -1238,7 +1305,7 @@
String tag,
String[] allowedAttrs,
String[] uriAttrs) {
- super(tag, allowedAttrs, uriAttrs);
+ super(tag, allowedAttrs, uriAttrs, null);
}
abstract void setStyle(boolean b, HTMLParseContext pc);
@@ -1384,8 +1451,9 @@
BaseCoreTagVerifier(
String tag,
String[] allowedAttrs,
- String[] uriAttrs) {
- super(tag, allowedAttrs, uriAttrs);
+ String[] uriAttrs,
+ String[] inlineURIAttrs) {
+ super(tag, allowedAttrs, uriAttrs, inlineURIAttrs);
}
Hashtable sanitizeHash(
@@ -1456,17 +1524,19 @@
String tag,
String[] allowedAttrs,
String[] uriAttrs,
+ String[] inlineURIAttrs,
String[] eventAttrs) {
- this(tag, allowedAttrs, uriAttrs, eventAttrs, true);
+ this(tag, allowedAttrs, uriAttrs, inlineURIAttrs,
eventAttrs, true);
}
CoreTagVerifier(
String tag,
String[] allowedAttrs,
String[] uriAttrs,
+ String[] inlineURIAttrs,
String[] eventAttrs,
boolean addStdEvents) {
- super(tag, allowedAttrs, uriAttrs);
+ super(tag, allowedAttrs, uriAttrs, inlineURIAttrs);
this.eventAttrs = new HashSet();
if (eventAttrs != null) {
for (int x = 0; x < eventAttrs.length; x++)
@@ -1503,8 +1573,9 @@
String tag,
String[] allowedAttrs,
String[] uriAttrs,
+ String[] inlineURIAttrs,
String[] eventAttrs) {
- super(tag, allowedAttrs, uriAttrs, eventAttrs);
+ super(tag, allowedAttrs, uriAttrs, inlineURIAttrs,
eventAttrs);
}
Hashtable sanitizeHash(
@@ -1555,7 +1626,7 @@
// type+"
and charset "+charset,
//
Logger.DEBUG);
href = HTMLDecoder.decode(href);
- href = htmlSanitizeURI(href, type, charset,
pc.cb, pc);
+ href = htmlSanitizeURI(href, type, charset,
pc.cb, pc, false);
if (href != null) {
href = HTMLEncoder.encode(href);
hn.put("href", href);
@@ -1585,7 +1656,7 @@
String[] allowedAttrs,
String[] uriAttrs,
String[] eventAttrs) {
- super(tag, allowedAttrs, uriAttrs, eventAttrs);
+ super(tag, allowedAttrs, uriAttrs, null, eventAttrs);
}
Hashtable sanitizeHash(
@@ -1635,8 +1706,9 @@
String tag,
String[] allowedAttrs,
String[] uriAttrs,
+ String[] inlineURIAttrs,
String[] eventAttrs) {
- super(tag, allowedAttrs, uriAttrs, eventAttrs);
+ super(tag, allowedAttrs, uriAttrs, inlineURIAttrs,
eventAttrs);
this.allowedTypes = new HashSet();
if (types != null) {
for (int x = 0; x < types.length; x++)
@@ -1836,7 +1908,7 @@
static class BaseHrefTagVerifier extends TagVerifier {
BaseHrefTagVerifier(String string, String[] strings, String[]
strings2) {
- super(string, strings, strings2);
+ super(string, strings, strings2, null);
}
Hashtable sanitizeHash(
@@ -1907,8 +1979,8 @@
return null;
}
- static String sanitizeURI(String uri, FilterCallback cb) throws
CommentException {
- return sanitizeURI(uri, null, null, cb);
+ static String sanitizeURI(String uri, FilterCallback cb, boolean
inline) throws CommentException {
+ return sanitizeURI(uri, null, null, cb, inline);
}
/*
@@ -1977,9 +2049,10 @@
String overrideType,
String overrideCharset,
FilterCallback cb,
- HTMLParseContext pc) {
+ HTMLParseContext pc,
+ boolean inline) {
try {
- return sanitizeURI(suri, overrideType, overrideCharset,
cb);
+ return sanitizeURI(suri, overrideType, overrideCharset,
cb, inline);
} catch (CommentException e) {
pc.writeAfterTag.append("<!--
").append(HTMLEncoder.encode(e.toString())).append(" -->");
return null;
@@ -1990,12 +2063,12 @@
String suri,
String overrideType,
String overrideCharset,
- FilterCallback cb) throws CommentException {
+ FilterCallback cb, boolean inline) throws CommentException {
if(logMINOR)
- Logger.minor(HTMLFilter.class, "Sanitizing URI:
"+suri+" ( override type "+overrideType +" override charset "+overrideCharset+"
)");
+ Logger.minor(HTMLFilter.class, "Sanitizing URI:
"+suri+" ( override type "+overrideType +" override charset "+overrideCharset+"
) inline="+inline, new Exception("debug"));
if((overrideCharset != null) && (overrideCharset.length() > 0))
overrideType += "; charset="+overrideCharset;
- return cb.processURI(suri, overrideType);
+ return cb.processURI(suri, overrideType, false, inline);
}
static String getHashString(Hashtable h, String key) {
Modified: trunk/freenet/src/freenet/clients/http/filter/NullFilterCallback.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/filter/NullFilterCallback.java
2008-03-05 19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/clients/http/filter/NullFilterCallback.java
2008-03-05 19:57:13 UTC (rev 18377)
@@ -21,4 +21,8 @@
return null;
}
+ public String processURI(String uri, String overrideType, boolean
noRelative, boolean inline) throws CommentException {
+ return null;
+ }
+
}
Modified: trunk/freenet/src/freenet/node/simulator/RealNodeULPRTest.java
===================================================================
--- trunk/freenet/src/freenet/node/simulator/RealNodeULPRTest.java
2008-03-05 19:49:02 UTC (rev 18376)
+++ trunk/freenet/src/freenet/node/simulator/RealNodeULPRTest.java
2008-03-05 19:57:13 UTC (rev 18377)
@@ -78,8 +78,8 @@
DummyRandomSource random = new DummyRandomSource();
//NOTE: globalTestInit returns in ignored random source
- NodeStarter.globalTestInit(testName, false, Logger.ERROR,
"freenet.node.Location:normal,freenet.node.simulator.RealNodeRoutingTest:normal,freenet.node.NodeDispatcher:NORMAL"
/*,freenet.node.FailureTable:MINOR,freenet.node.Node:MINOR,freenet.node.Request:MINOR,freenet.io.comm.MessageCore:MINOR"
"freenet.store:minor,freenet.node.LocationManager:debug,freenet.node.FNPPacketManager:normal,freenet.io.comm.MessageCore:debug"*/);
- //NodeStarter.globalTestInit(testName, false, Logger.ERROR,
"freenet.node.Location:normal,freenet.node.simulator.RealNodeRoutingTest:normal,freenet.node.NodeDispatcher:NORMAL,freenet.node.FailureTable:MINOR,freenet.node.Node:MINOR,freenet.node.Request:MINOR,freenet.io.comm.MessageCore:MINOR,freenet.node.PeerNode:MINOR,freenet.io.xfer.PacketThrottle:MINOR");
+ //NodeStarter.globalTestInit(testName, false, Logger.ERROR,
"freenet.node.Location:normal,freenet.node.simulator.RealNodeRoutingTest:normal,freenet.node.NodeDispatcher:NORMAL"
/*,freenet.node.FailureTable:MINOR,freenet.node.Node:MINOR,freenet.node.Request:MINOR,freenet.io.comm.MessageCore:MINOR"
"freenet.store:minor,freenet.node.LocationManager:debug,freenet.node.FNPPacketManager:normal,freenet.io.comm.MessageCore:debug"*/);
+ NodeStarter.globalTestInit(testName, false, Logger.ERROR,
"freenet.node.Location:normal,freenet.node.simulator.RealNodeRoutingTest:normal,freenet.node.NodeDispatcher:NORMAL,freenet.node.FailureTable:MINOR,freenet.node.Node:MINOR,freenet.node.Request:MINOR,freenet.io.comm.MessageCore:MINOR,freenet.node.PeerNode:MINOR,freenet.io.xfer.PacketThrottle:MINOR");
Node[] nodes = new Node[NUMBER_OF_NODES];
Logger.normal(RealNodeRoutingTest.class, "Creating nodes...");
Executor executor = new PooledExecutor();
Modified: trunk/plugins/XMLSpider/XMLSpider.java
===================================================================
--- trunk/plugins/XMLSpider/XMLSpider.java 2008-03-05 19:49:02 UTC (rev
18376)
+++ trunk/plugins/XMLSpider/XMLSpider.java 2008-03-05 19:57:13 UTC (rev
18377)
@@ -1138,6 +1138,10 @@
}
public void foundURI(FreenetURI uri){
+ // Ignore
+ }
+
+ public void foundURI(FreenetURI uri, boolean inline){
Logger.minor(this, "foundURI "+uri+" on "+id);
queueURI(uri);
