Author: toad
Date: 2007-05-31 19:19:39 +0000 (Thu, 31 May 2007)
New Revision: 13416
Modified:
trunk/freenet/src/freenet/clients/http/FProxyToadlet.java
trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java
trunk/freenet/src/freenet/clients/http/ToadletContainer.java
trunk/freenet/src/freenet/clients/http/ToadletContext.java
trunk/freenet/src/freenet/clients/http/ToadletContextImpl.java
Log:
Configurable doRobots parameter. Default to false.
Modified: trunk/freenet/src/freenet/clients/http/FProxyToadlet.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/FProxyToadlet.java 2007-05-31
13:27:11 UTC (rev 13415)
+++ trunk/freenet/src/freenet/clients/http/FProxyToadlet.java 2007-05-31
19:19:39 UTC (rev 13416)
@@ -317,7 +317,7 @@
ctx.writeData(buf, 0, len);
}
return;
- }else if(ks.equals("/robots.txt")){
+ }else if(ks.equals("/robots.txt") && ctx.doRobots()){
this.writeReply(ctx, 200, "text/plain", "Ok",
"User-agent: *\nDisallow: /");
return;
}else if(ks.startsWith("/darknet/")) { //TODO: remove when
obsolete
Modified: trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java
2007-05-31 13:27:11 UTC (rev 13415)
+++ trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java
2007-05-31 19:19:39 UTC (rev 13416)
@@ -61,6 +61,7 @@
private boolean fProxyJavascriptEnabled;
private final PageMaker pageMaker;
private final NodeClientCore core;
+ private boolean doRobots;
static boolean isPanicButtonToBeShown;
static final int DEFAULT_FPROXY_PORT = 8888;
@@ -310,6 +311,16 @@
});
allowedFullAccess = new
AllowedHosts(fproxyConfig.getString("allowedHostsFullAccess"));
+ fproxyConfig.register("doRobots", false, configItemOrder++,
true, false, "Exclude robots via robots.txt?", "Whether to serve a /robots.txt
telling google, spiders, wget, etc to go away",
+ new BooleanCallback() {
+ public boolean get() {
+ return doRobots;
+ }
+ public void set(boolean val) throws
InvalidConfigValueException {
+ doRobots = val;
+ }
+ });
+ doRobots = fproxyConfig.getBoolean("doRobots");
SimpleToadletServer.isPanicButtonToBeShown =
fproxyConfig.getBoolean("showPanicButton");
this.bf = core.tempBucketFactory;
@@ -340,6 +351,10 @@
}
}
+ public boolean doRobots() {
+ return doRobots;
+ }
+
public void start() {
if(myThread != null) {
myThread.start();
Modified: trunk/freenet/src/freenet/clients/http/ToadletContainer.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/ToadletContainer.java
2007-05-31 13:27:11 UTC (rev 13415)
+++ trunk/freenet/src/freenet/clients/http/ToadletContainer.java
2007-05-31 19:19:39 UTC (rev 13416)
@@ -33,4 +33,7 @@
/** Is the given IP address allowed full access to the node? */
public boolean isAllowedFullAccess(InetAddress remoteAddr);
+
+ /** Whether to tell spiders to go away */
+ public boolean doRobots();
}
Modified: trunk/freenet/src/freenet/clients/http/ToadletContext.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/ToadletContext.java 2007-05-31
13:27:11 UTC (rev 13415)
+++ trunk/freenet/src/freenet/clients/http/ToadletContext.java 2007-05-31
19:19:39 UTC (rev 13416)
@@ -67,5 +67,10 @@
/** Is this Toadlet allowed full access to the node, including the
ability to reconfigure it,
* restart it etc? */
boolean isAllowedFullAccess();
+
+ /**
+ * Return a robots.txt excluding all spiders and other non-browser HTTP
clients?
+ */
+ boolean doRobots();
}
Modified: trunk/freenet/src/freenet/clients/http/ToadletContextImpl.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/ToadletContextImpl.java
2007-05-31 13:27:11 UTC (rev 13415)
+++ trunk/freenet/src/freenet/clients/http/ToadletContextImpl.java
2007-05-31 19:19:39 UTC (rev 13416)
@@ -435,4 +435,8 @@
public boolean isAllowedFullAccess() {
return container.isAllowedFullAccess(remoteAddr);
}
+
+ public boolean doRobots() {
+ return container.doRobots();
+ }
}
