Author: ab
Date: Wed Aug 18 12:36:15 2010
New Revision: 986648
URL: http://svn.apache.org/viewvc?rev=986648&view=rev
Log:
Some improvements to FakeHandler to make the fake pages more
predictable and to follow a few predefined spread modes.
Modified:
nutch/trunk/build.xml
nutch/trunk/src/java/org/apache/nutch/tools/proxy/FakeHandler.java
nutch/trunk/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java
Modified: nutch/trunk/build.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/build.xml?rev=986648&r1=986647&r2=986648&view=diff
==============================================================================
--- nutch/trunk/build.xml (original)
+++ nutch/trunk/build.xml Wed Aug 18 12:36:15 2010
@@ -225,6 +225,11 @@
<java classname="org.apache.nutch.tools.proxy.TestbedProxy" fork="true">
<classpath refid="test.classpath"/>
<arg value="-fake"/>
+ <arg value="-hostMode"/>
+ <arg value="u"/>
+ <arg value="-pageMode"/>
+ <arg value="u"/>
+ <arg value="-debug"/>
<!--
<arg value="-delay"/>
<arg value="-200"/>
Modified: nutch/trunk/src/java/org/apache/nutch/tools/proxy/FakeHandler.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/tools/proxy/FakeHandler.java?rev=986648&r1=986647&r2=986648&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/tools/proxy/FakeHandler.java
(original)
+++ nutch/trunk/src/java/org/apache/nutch/tools/proxy/FakeHandler.java Wed Aug
18 12:36:15 2010
@@ -19,6 +19,7 @@ package org.apache.nutch.tools.proxy;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletResponse;
@@ -27,7 +28,20 @@ import org.mortbay.jetty.HttpURI;
import org.mortbay.jetty.Request;
public class FakeHandler extends AbstractTestbedHandler {
+ /** Create links to hosts generated from a pool of numHosts/numPages random
names. */
+ public static enum Mode {UNIQUE, RANDOM};
+
+ int numInternalLinks;
+ int numExternalLinks;
+ Mode hostMode;
+ Mode pageMode;
+ AtomicLong hostSeq = new AtomicLong(0);
+ AtomicLong pageSeq = new AtomicLong(0);
+ int numHosts;
+ int numPages;
+
Random r = new Random(1234567890L); // predictable
+ Random pageR;
private static final String testA =
"<html><body><h1>Internet Weather Forecast Accuracy</h1>\n" +
@@ -39,12 +53,33 @@ public class FakeHandler extends Abstrac
"<p>The hail, rain and lightning eventually subsided, but the most
alarming news was waiting on cell phone voicemail. A friend who lived in the
area had called frantically, knowing we were at the park, as the local news was
reporting multiple people had been by struck by lightning at Schlitterbahn
during the storm.</p>" +
"<p>'So much for the 0% chance of rain,' I repeated.</p></body></html>";
+ /**
+ * Create fake pages.
+ * @param hostMode if UNIQUE then each external outlink will use a unique
host name. If
+ * RANDOM then each outlink will use a host name allocated from pool of
numHosts.
+ * @param pageMode if UNIQUE then each internal outlinks will use a unique
page name.
+ * if RANDOM then each outlink will use a page name allocated from pool of
numPages.
+ * @param numInternalLinks
+ * @param numExternalLinks
+ * @param numHosts
+ * @param numPages
+ */
+ public FakeHandler(Mode hostMode, Mode pageMode,
+ int numInternalLinks, int numExternalLinks,
+ int numHosts, int numPages) {
+ this.numExternalLinks = numExternalLinks;
+ this.numInternalLinks = numInternalLinks;
+ this.numHosts = numHosts;
+ this.numPages = numPages;
+ this.hostMode = hostMode;
+ this.pageMode = pageMode;
+ }
+
@Override
public void handle(Request req, HttpServletResponse res, String target,
int dispatch) throws IOException, ServletException {
HttpURI u = req.getUri();
String uri = u.toString();
- //System.err.println("-faking " + uri.toString());
addMyHeader(res, "URI", uri);
// don't pass it down the chain
req.setHandled(true);
@@ -61,32 +96,51 @@ public class FakeHandler extends Abstrac
String p = "<p>URI: " + uri + "</p>\r\n";
os.write(p.getBytes());
// fake some links
- String base;
+ String basePath;
+ String baseDomain;
if (u.getPath().length() > 5) {
- base = u.getPath().substring(0, u.getPath().length() - 5);
+ basePath = u.getPath().substring(0, u.getPath().length() - 5);
} else {
- base = u.getPath();
+ basePath = u.getPath();
+ }
+ // internal links
+ if (pageMode.equals(Mode.RANDOM)) { // initialize random per host
+ pageR = new Random(u.getHost().hashCode());
}
- String prefix = u.getScheme() + "://" + u.getHost();
- if (u.getPort() != 80 && u.getPort() != -1) base += ":" + u.getPort();
- if (!base.startsWith("/")) prefix += "/";
- prefix = prefix + base;
- for (int i = 0; i < 10; i++) {
- String link = "<p><a href='" + prefix;
- if (!prefix.endsWith("/")) {
- link += "/";
+ for (int i = 0; i < numInternalLinks; i++) {
+ String link = "<p><a href='";
+ if (pageMode.equals(Mode.RANDOM)) {
+ link += pageR.nextInt (numPages) + ".html'>";
+ } else {
+ if (!basePath.endsWith("/")) {
+ link += "/";
+ }
+ link += pageSeq.getAndIncrement() + ".html'>";
}
- link += i + ".html'>outlink " + i + "</a></p>\r\n";
+ link += "outlink " + i + "</a></p>\r\n";
os.write(link.getBytes());
}
- // fake a few links to random nonexistent hosts
- for (int i = 0; i < 5; i++) {
- int h = r.nextInt(1000000); // 1 mln hosts
- String link = "<p><a href='http://www.fake-"; + h + ".com/'>fake host "
+ h + "</a></p>\r\n";
+ baseDomain = u.getHost();
+ // chop off the TLD
+ int pos = baseDomain.lastIndexOf('.');
+ String tld = baseDomain.substring(pos);
+ baseDomain = baseDomain.substring(0, pos);
+ String link;
+ // external links
+ for (int i = 0; i < numExternalLinks; i++) {
+ String host;
+ if (hostMode.equals(Mode.RANDOM)) {
+ host = "www.rnd-" + r.nextInt(numHosts) + ".com";
+ link = "http://"; + host + "/";
+ } else {
+ host = baseDomain + "-" + hostSeq.getAndIncrement() + ".com";
+ link = "http://"; + host + "/";
+ }
+ link = "<p><a href='" + link + "'>fake host " + host + "</a></p>\r\n";
os.write(link.getBytes());
}
// fake a link to the root URL
- String link = "<p><a href='" + u.getScheme() + "://" + u.getHost();
+ link = "<p><a href='" + u.getScheme() + "://" + u.getHost();
if (u.getPort() != 80 && u.getPort() != -1) link += ":" + u.getPort();
link += "/'>site " + u.getHost() + "</a></p>\r\n";
os.write(link.getBytes());
Modified: nutch/trunk/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java?rev=986648&r1=986647&r2=986648&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java
(original)
+++ nutch/trunk/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java Wed Aug
18 12:36:15 2010
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
+import org.apache.nutch.tools.proxy.FakeHandler.Mode;
import org.apache.nutch.util.HadoopFSUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.mortbay.jetty.Handler;
@@ -44,12 +45,26 @@ public class TestbedProxy {
*/
public static void main(String[] args) throws Exception {
if (args.length == 0) {
- System.err.println("TestbedProxy [-port <nnn>] [-forward] [-fake]
[-delay nnn] [-debug]");
+ System.err.println("TestbedProxy [-port <nnn>] [-forward] [-fake [...]]
[-delay nnn] [-debug]");
System.err.println("-port <nnn>\trun the proxy on port <nnn> (special
permissions may be needed for ports < 1024)");
System.err.println("-forward\tif specified, requests to all unknown urls
will be passed to");
System.err.println("\t\toriginal servers. If false (default) unknown
urls generate 404 Not Found.");
System.err.println("-delay\tdelay every response by nnn seconds. If
delay is negative use a random value up to nnn");
System.err.println("-fake\tif specified, requests to all unknown urls
will succeed with fake content");
+ System.err.println("\nAdditional options for -fake handler (all
optional):");
+ System.err.println("\t-hostMode (u | r)\tcreate unique host names, or
pick random from a pool");
+ System.err.println("\t-pageMode (u | r)\tcreate unique page names, or
pick random from a pool");
+ System.err.println("\t-numHosts N\ttotal number of hosts when using
hostMode r");
+ System.err.println("\t-numPages N\ttotal number of pages per host when
using pageMode r");
+ System.err.println("\t-intLinks N\tnumber of internal (same host) links
per page");
+ System.err.println("\t-extLinks N\tnumber of external (other host) links
per page");
+ System.err.println("\nDefaults for -fake handler:");
+ System.err.println("\t-hostMode r");
+ System.err.println("\t-pageMode r");
+ System.err.println("\t-numHosts 1000000");
+ System.err.println("\t-numPages 10000");
+ System.err.println("\t-intLinks 10");
+ System.err.println("\t-extLinks 5");
System.exit(-1);
}
@@ -60,6 +75,12 @@ public class TestbedProxy {
boolean delay = false;
boolean debug = false;
int delayVal = 0;
+ Mode pageMode = Mode.RANDOM;
+ Mode hostMode = Mode.RANDOM;
+ int numHosts = 1000000;
+ int numPages = 10000;
+ int intLinks = 10;
+ int extLinks = 5;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-port")) {
@@ -71,6 +92,22 @@ public class TestbedProxy {
delayVal = Integer.parseInt(args[++i]);
} else if (args[i].equals("-fake")) {
fake = true;
+ } else if (args[i].equals("-hostMode")) {
+ if (args[++i].equals("u")) {
+ hostMode = Mode.UNIQUE;
+ }
+ } else if (args[i].equals("-pageMode")) {
+ if (args[++i].equals("u")) {
+ pageMode = Mode.UNIQUE;
+ }
+ } else if (args[i].equals("-numHosts")) {
+ numHosts = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("-numPages")) {
+ numPages = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("-intLinks")) {
+ intLinks = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("-extLinks")) {
+ extLinks = Integer.parseInt(args[++i]);
} else if (args[i].equals("-debug")) {
debug = true;
} else {
@@ -113,7 +150,8 @@ public class TestbedProxy {
}
if (fake) {
LOG.info("* Added fake handler for remaining URLs.");
- list.addHandler(new FakeHandler());
+ list.addHandler(new FakeHandler(hostMode, pageMode, intLinks, extLinks,
+ numHosts, numPages));
}
list.addHandler(new NotFoundHandler());
// Start the http server
