Author: ab
Date: Thu Aug 5 12:51:04 2010
New Revision: 982588
URL: http://svn.apache.org/viewvc?rev=982588&view=rev
Log:
NUTCH-867 Port Nutch benchmark to Nutchbase.
Added:
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/Benchmark.java
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/AbstractTestbedHandler.java
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/DelayHandler.java
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/FakeHandler.java
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/LogDebugHandler.java
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/NotFoundHandler.java
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java
Modified:
nutch/branches/nutchbase/CHANGES.txt
nutch/branches/nutchbase/build.xml
nutch/branches/nutchbase/ivy/ivy.xml
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserJob.java
Modified: nutch/branches/nutchbase/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/CHANGES.txt?rev=982588&r1=982587&r2=982588&view=diff
==============================================================================
--- nutch/branches/nutchbase/CHANGES.txt (original)
+++ nutch/branches/nutchbase/CHANGES.txt Thu Aug 5 12:51:04 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.0 - Current Development
+* NUTCH-867 Port Nutch benchmark to Nutchbase (ab)
+
* NUTCH-869 Add parse-html back (jnioche)
* NUTCH-871 MoreIndexingFilter missing date format (Max Lynch via
mattmann)
Modified: nutch/branches/nutchbase/build.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/build.xml?rev=982588&r1=982587&r2=982588&view=diff
==============================================================================
--- nutch/branches/nutchbase/build.xml (original)
+++ nutch/branches/nutchbase/build.xml Thu Aug 5 12:51:04 2010
@@ -218,6 +218,42 @@
<fail if="pmd.stop">FAILURE: PMD shows ${pmd.failures} rule violations.
See ${pmd.report} for details.</fail>
</target>
+ <target name="proxy" depends="job">
+ <java classname="org.apache.nutch.tools.proxy.TestbedProxy" fork="true">
+ <classpath refid="test.classpath"/>
+ <arg value="-fake"/>
+<!--
+ <arg value="-delay"/>
+ <arg value="-200"/>
+-->
+ <jvmarg
line="-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
+ </java>
+ </target>
+
+ <target name="rundb">
+ <java classname="org.hsqldb.server.Server" fork="true">
+ <classpath refid="test.classpath"/>
+ <arg value="--database.0"/>
+ <arg value="file:data/benchmark"/>
+ <arg value="--dbname.0"/>
+ <arg value="nutchtest"/>
+ </java>
+ </target>
+
+ <target name="benchmark">
+ <java classname="org.apache.nutch.tools.Benchmark" fork="true">
+ <classpath refid="test.classpath"/>
+ <jvmarg line="-Xmx512m
-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
+ <arg value="-maxPerHost"/>
+ <arg value="10"/>
+ <arg value="-seeds"/>
+ <arg value="1"/>
+ <arg value="-depth"/>
+ <arg value="5"/>
+ </java>
+ </target>
+
+
<!-- ================================================================== -->
<!-- Run unit tests -->
<!-- ================================================================== -->
@@ -271,13 +307,13 @@
<!-- ================================================================== -->
<!-- target: resolve ================================================= -->
- <target name="resolve-default" depends="clean-lib" description="--> resolve
and retrieve dependencies with ivy">
+ <target name="resolve-default" depends="clean-lib, init" description="-->
resolve and retrieve dependencies with ivy">
<ivy:resolve file="${ivy.file}" conf="default" log="download-only"/>
<ivy:retrieve pattern="${build.lib.dir}/[artifact]-[revision].[ext]"
symlink="false" log="quiet"/>
<antcall target="copy-libs"/>
</target>
- <target name="resolve-test" depends="clean-lib" description="--> resolve and
retrieve dependencies with ivy">
+ <target name="resolve-test" depends="clean-lib, init" description="-->
resolve and retrieve dependencies with ivy">
<ivy:resolve file="${ivy.file}" conf="test" log="download-only"/>
<ivy:retrieve pattern="${build.lib.dir}/[artifact]-[revision].[ext]"
symlink="false" log="quiet"/>
<antcall target="copy-libs"/>
Modified: nutch/branches/nutchbase/ivy/ivy.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/ivy/ivy.xml?rev=982588&r1=982587&r2=982588&view=diff
==============================================================================
--- nutch/branches/nutchbase/ivy/ivy.xml (original)
+++ nutch/branches/nutchbase/ivy/ivy.xml Thu Aug 5 12:51:04 2010
@@ -104,6 +104,7 @@
conf="test->default" />
<dependency org="org.mortbay.jetty" name="jetty-util"
rev="6.1.22"
conf="test->default" />
+ <dependency org="org.mortbay.jetty" name="jetty-client"
rev="6.1.22" />
<dependency org="org.hsqldb" name="hsqldb" rev="2.0.0"
conf="*->default"/>
<dependency org="org.jdom" name="jdom" rev="1.1"
conf="test->default"/>
Modified:
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/DbUpdaterJob.java?rev=982588&r1=982587&r2=982588&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
(original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/DbUpdaterJob.java
Thu Aug 5 12:51:04 2010
@@ -5,6 +5,7 @@ import java.util.HashSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.mapreduce.Job;
@@ -41,6 +42,14 @@ implements Tool {
FIELDS.add(WebPage.Field.PREV_FETCH_TIME);
FIELDS.add(WebPage.Field.PREV_SIGNATURE);
}
+
+ public DbUpdaterJob() {
+
+ }
+
+ public DbUpdaterJob(Configuration conf) {
+ setConf(conf);
+ }
private int updateTable() throws Exception {
LOG.info("DbUpdaterJob: starting");
Modified:
nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserJob.java?rev=982588&r1=982587&r2=982588&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserJob.java
(original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/parse/ParserJob.java Thu
Aug 5 12:51:04 2010
@@ -54,6 +54,14 @@ public class ParserJob extends GoraMappe
private boolean force;
private Utf8 crawlId;
+
+ public ParserJob() {
+
+ }
+
+ public ParserJob(Configuration conf) {
+ setConf(conf);
+ }
@Override
public void setup(Context context) throws IOException {
Added: nutch/branches/nutchbase/src/java/org/apache/nutch/tools/Benchmark.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/tools/Benchmark.java?rev=982588&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/tools/Benchmark.java
(added)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/tools/Benchmark.java Thu
Aug 5 12:51:04 2010
@@ -0,0 +1,144 @@
+package org.apache.nutch.tools;
+
+import java.io.OutputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.DbUpdaterJob;
+import org.apache.nutch.crawl.GeneratorJob;
+import org.apache.nutch.crawl.InjectorJob;
+import org.apache.nutch.crawl.WebTableReader;
+import org.apache.nutch.fetcher.FetcherJob;
+import org.apache.nutch.parse.ParserJob;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.NutchJob;
+
+public class Benchmark extends Configured implements Tool {
+ private static final Log LOG = LogFactory.getLog(Benchmark.class);
+
+ public static void main(String[] args) throws Exception {
+ Configuration conf = NutchConfiguration.create();
+ int res = ToolRunner.run(conf, new Benchmark(), args);
+ System.exit(res);
+ }
+
+ private void createSeeds(FileSystem fs, Path seedsDir, int count) throws
Exception {
+ OutputStream os = fs.create(new Path(seedsDir, "seeds"));
+ for (int i = 0; i < count; i++) {
+ String url = "http://www.test-"; + i + ".com/\r\n";
+ os.write(url.getBytes());
+ }
+ os.flush();
+ os.close();
+ }
+
+ public int run(String[] args) throws Exception {
+ String plugins =
"protocol-http|parse-tika|scoring-opic|urlfilter-regex|urlnormalizer-pass";
+ int seeds = 1;
+ int depth = 10;
+ int threads = 10;
+ //boolean delete = true;
+ long topN = Long.MAX_VALUE;
+
+ if (args.length == 0) {
+ System.err.println("Usage: Benchmark [-seeds NN] [-depth NN] [-threads
NN] [-maxPerHost NN] [-plugins <regex>]");
+ System.err.println("\t-seeds NN\tcreate NN unique hosts in a seed list
(default: 1)");
+ System.err.println("\t-depth NN\tperform NN crawl cycles (default: 10)");
+ System.err.println("\t-threads NN\tuse NN threads per Fetcher task
(default: 10)");
+ // XXX what is the equivalent here? not an additional job...
+ // System.err.println("\t-keep\tkeep segment data (default: delete after
updatedb)");
+ System.err.println("\t-plugins <regex>\toverride 'plugin.includes'.");
+ System.err.println("\tNOTE: if not specified, this is reset to: " +
plugins);
+ System.err.println("\tNOTE: if 'default' is specified then a value set
in nutch-default/nutch-site is used.");
+ System.err.println("\t-maxPerHost NN\tmax. # of URLs per host in a
fetchlist");
+ return -1;
+ }
+ int maxPerHost = Integer.MAX_VALUE;
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].equals("-seeds")) {
+ seeds = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("-threads")) {
+ threads = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("-depth")) {
+ depth = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("-plugins")) {
+ plugins = args[++i];
+ } else if (args[i].equalsIgnoreCase("-maxPerHost")) {
+ maxPerHost = Integer.parseInt(args[++i]);
+ } else {
+ LOG.fatal("Invalid argument: '" + args[i] + "'");
+ return -1;
+ }
+ }
+ Configuration conf = getConf();
+ conf.set("http.proxy.host", "localhost");
+ conf.setInt("http.proxy.port", 8181);
+ conf.set("http.agent.name", "test");
+ conf.set("http.robots.agents", "test,*");
+ if (!plugins.equals("default")) {
+ conf.set("plugin.includes", plugins);
+ }
+ conf.setInt(GeneratorJob.GENERATOR_MAX_COUNT, maxPerHost);
+ conf.set(GeneratorJob.GENERATOR_COUNT_MODE,
GeneratorJob.GENERATOR_COUNT_VALUE_HOST);
+ Job job = new NutchJob(conf);
+ FileSystem fs = FileSystem.get(job.getConfiguration());
+ Path dir = new Path(getConf().get("hadoop.tmp.dir"),
+ "bench-" + System.currentTimeMillis());
+ fs.mkdirs(dir);
+ Path rootUrlDir = new Path(dir, "seed");
+ fs.mkdirs(rootUrlDir);
+ createSeeds(fs, rootUrlDir, seeds);
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("crawl started in: " + dir);
+ LOG.info("rootUrlDir = " + rootUrlDir);
+ LOG.info("threads = " + threads);
+ LOG.info("depth = " + depth);
+ }
+
+ long start = System.currentTimeMillis();
+ InjectorJob injector = new InjectorJob(conf);
+ GeneratorJob generator = new GeneratorJob(conf);
+ FetcherJob fetcher = new FetcherJob(conf);
+ ParserJob parseSegment = new ParserJob(conf);
+ DbUpdaterJob crawlDbTool = new DbUpdaterJob(conf);
+ // not needed in the new API
+ //LinkDb linkDbTool = new LinkDb(getConf());
+
+ // initialize crawlDb
+ injector.inject(rootUrlDir);
+ int i;
+ for (i = 0; i < depth; i++) { // generate new segment
+ String crawlId = generator.generate(topN, System.currentTimeMillis(),
+ false, false);
+ if (crawlId == null) {
+ LOG.info("Stopping at depth=" + i + " - no more URLs to fetch.");
+ break;
+ }
+ boolean isParsing = getConf().getBoolean("fetcher.parse", true);
+ fetcher.fetch(threads, crawlId, false, isParsing); // fetch it
+ if (!isParsing) {
+ parseSegment.parse(crawlId, false, false); // parse it, if needed
+ }
+ crawlDbTool.run(new String[0]); // update crawldb
+ }
+ if (i == 0) {
+ LOG.warn("No URLs to fetch - check your seed list and URL filters.");
+ }
+ if (LOG.isInfoEnabled()) { LOG.info("crawl finished: " + dir); }
+ long end = System.currentTimeMillis();
+ LOG.info("TOTAL TIME: " + (end - start)/1000 + " sec");
+ WebTableReader dbreader = new WebTableReader();
+ dbreader.setConf(conf);
+ dbreader.processStatJob(false);
+ return 0;
+ }
+
+}
Added:
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/AbstractTestbedHandler.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/AbstractTestbedHandler.java?rev=982588&view=auto
==============================================================================
---
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/AbstractTestbedHandler.java
(added)
+++
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/AbstractTestbedHandler.java
Thu Aug 5 12:51:04 2010
@@ -0,0 +1,47 @@
+package org.apache.nutch.tools.proxy;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.mortbay.jetty.HttpConnection;
+import org.mortbay.jetty.Request;
+import org.mortbay.jetty.handler.AbstractHandler;
+
+public abstract class AbstractTestbedHandler extends AbstractHandler {
+ protected boolean debug = false;
+
+ @Override
+ public void handle(String target, HttpServletRequest req,
+ HttpServletResponse res, int dispatch) throws IOException,
+ ServletException {
+ Request base_request = (req instanceof Request) ? (Request)req :
HttpConnection.getCurrentConnection().getRequest();
+ res.addHeader("X-TestbedHandlers", this.getClass().getSimpleName());
+ handle(base_request, res, target, dispatch);
+ }
+
+ public abstract void handle(Request req, HttpServletResponse res, String
target,
+ int dispatch) throws IOException, ServletException;
+
+ public void addMyHeader(HttpServletResponse res, String name, String value) {
+ name = "X-" + this.getClass().getSimpleName() + "-" + name;
+ res.addHeader(name, value);
+ }
+}
Added:
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/DelayHandler.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/DelayHandler.java?rev=982588&view=auto
==============================================================================
---
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/DelayHandler.java
(added)
+++
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/DelayHandler.java
Thu Aug 5 12:51:04 2010
@@ -0,0 +1,55 @@
+package org.apache.nutch.tools.proxy;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Random;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletResponse;
+
+import org.mortbay.jetty.Request;
+
+public class DelayHandler extends AbstractTestbedHandler {
+
+ public static final long DEFAULT_DELAY = 2000;
+
+ private int delay;
+ private boolean random;
+ private Random r;
+
+ public DelayHandler(int delay) {
+ if (delay < 0) {
+ delay = -delay;
+ random = true;
+ r = new Random(1234567890L); // repeatable random
+ }
+ this.delay = delay;
+ }
+
+ @Override
+ public void handle(Request req, HttpServletResponse res, String target,
+ int dispatch) throws IOException, ServletException {
+ try {
+ int del = random ? r.nextInt(delay) : delay;
+ Thread.sleep(del);
+ addMyHeader(res, "Delay", String.valueOf(del));
+ } catch (Exception e) {
+
+ }
+ }
+}
Added:
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/FakeHandler.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/FakeHandler.java?rev=982588&view=auto
==============================================================================
---
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/FakeHandler.java
(added)
+++
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/FakeHandler.java
Thu Aug 5 12:51:04 2010
@@ -0,0 +1,99 @@
+package org.apache.nutch.tools.proxy;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Random;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletResponse;
+
+import org.mortbay.jetty.HttpURI;
+import org.mortbay.jetty.Request;
+
+public class FakeHandler extends AbstractTestbedHandler {
+ Random r = new Random(1234567890L); // predictable
+
+ private static final String testA =
+ "<html><body><h1>Internet Weather Forecast Accuracy</h1>\n" +
+ "<p>Weather forecasting is a secure and popular online presence, which is
understandable. The weather affects most everyone's life, and the Internet can
provide information on just about any location at any hour of the day or night.
But how accurate is this information? How much can we trust it? Perhaps it is
just my skeptical nature (or maybe the seeming unpredictability of nature), but
I've never put much weight into weather forecasts - especially those made more
than three days in advance. That skepticism progressed to a new high in the
Summer of 2004, but I have only now done the research necessary to test the
accuracy of online weather forecasts. First the story, then the data.</p>" +
+ "<h2>An Internet Weather Forecast Gone Terribly Awry</h2>" +
+ "<p>It was the Summer of 2004 and my wife and I were gearing up for a trip
with another couple to Schlitterbahn in New Braunfels - one of the (if not the)
best waterparks ever created. As a matter of course when embarking on a
2.5-hour drive to spend the day in a swimsuit, and given the tendency of the
area for natural disasters, we checked the weather. The temperatures looked
ideal and, most importantly, the chance of rain was a nice round goose
egg.</p>";
+ private static final String testB =
+ "<p>A couple of hours into our Schlitterbahn experience, we got on a bus
to leave the 'old section' for the 'new section.' Along the way, clouds
gathered and multiple claps of thunder sounded. 'So much for the 0% chance of
rain,' I commented. By the time we got to our destination, lightning sightings
had led to the slides and pools being evacuated and soon the rain began coming
down in torrents - accompanied by voluminous lightning flashes. After at least
a half an hour the downpour had subsided, but the lightning showed no sign of
letting up, so we began heading back to our vehicles. A hundred yards into the
parking lot, we passing a tree that had apparently been split in two during the
storm (whether by lightning or wind, I'm not sure). Not but a few yards later,
there was a distinct thud and the husband of the couple accompanying us cried
out as a near racquetball sized hunk of ice rebounded off of his head and onto
the concrete. Soon, similarly sized hail was falling
all around us as everyone scampered for cover. Some cowered under overturned
trashcans while others were more fortunate and made it indoors.</p>" +
+ "<p>The hail, rain and lightning eventually subsided, but the most
alarming news was waiting on cell phone voicemail. A friend who lived in the
area had called frantically, knowing we were at the park, as the local news was
reporting multiple people had been by struck by lightning at Schlitterbahn
during the storm.</p>" +
+ "<p>'So much for the 0% chance of rain,' I repeated.</p></body></html>";
+
+ @Override
+ public void handle(Request req, HttpServletResponse res, String target,
+ int dispatch) throws IOException, ServletException {
+ HttpURI u = req.getUri();
+ String uri = u.toString();
+ //System.err.println("-faking " + uri.toString());
+ addMyHeader(res, "URI", uri);
+ // don't pass it down the chain
+ req.setHandled(true);
+ res.addHeader("X-Handled-By", getClass().getSimpleName());
+ if (uri.endsWith("/robots.txt")) {
+ return;
+ }
+ res.setContentType("text/html");
+ try {
+ OutputStream os = res.getOutputStream();
+ byte[] bytes = testA.getBytes("UTF-8");
+ os.write(bytes);
+ // record URI
+ String p = "<p>URI: " + uri + "</p>\r\n";
+ os.write(p.getBytes());
+ // fake some links
+ String base;
+ if (u.getPath().length() > 5) {
+ base = u.getPath().substring(0, u.getPath().length() - 5);
+ } else {
+ base = u.getPath();
+ }
+ String prefix = u.getScheme() + "://" + u.getHost();
+ if (u.getPort() != 80 && u.getPort() != -1) base += ":" + u.getPort();
+ if (!base.startsWith("/")) prefix += "/";
+ prefix = prefix + base;
+ for (int i = 0; i < 10; i++) {
+ String link = "<p><a href='" + prefix;
+ if (!prefix.endsWith("/")) {
+ link += "/";
+ }
+ link += i + ".html'>outlink " + i + "</a></p>\r\n";
+ os.write(link.getBytes());
+ }
+ // fake a few links to random nonexistent hosts
+ for (int i = 0; i < 5; i++) {
+ int h = r.nextInt(1000000); // 1 mln hosts
+ String link = "<p><a href='http://www.fake-"; + h + ".com/'>fake host "
+ h + "</a></p>\r\n";
+ os.write(link.getBytes());
+ }
+ // fake a link to the root URL
+ String link = "<p><a href='" + u.getScheme() + "://" + u.getHost();
+ if (u.getPort() != 80 && u.getPort() != -1) link += ":" + u.getPort();
+ link += "/'>site " + u.getHost() + "</a></p>\r\n";
+ os.write(link.getBytes());
+ os.write(testB.getBytes());
+ res.flushBuffer();
+ } catch (IOException ioe) {
+ }
+ }
+
+}
Added:
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/LogDebugHandler.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/LogDebugHandler.java?rev=982588&view=auto
==============================================================================
---
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/LogDebugHandler.java
(added)
+++
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/LogDebugHandler.java
Thu Aug 5 12:51:04 2010
@@ -0,0 +1,59 @@
+package org.apache.nutch.tools.proxy;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.mortbay.jetty.Request;
+
+public class LogDebugHandler extends AbstractTestbedHandler implements Filter {
+ private static final Log LOG = LogFactory.getLog(LogDebugHandler.class);
+
+ @Override
+ public void handle(Request req, HttpServletResponse res, String target,
+ int dispatch) throws IOException, ServletException {
+ LOG.info("-- " + req.getMethod() + " " + req.getUri().toString() + "\n" +
req.getConnection().getRequestFields());
+ }
+
+ @Override
+ public void doFilter(ServletRequest req, ServletResponse res,
+ FilterChain chain) throws IOException, ServletException {
+ ((HttpServletResponse)res).addHeader("X-Handled-By", "AsyncProxyHandler");
+ ((HttpServletResponse)res).addHeader("X-TestbedHandlers",
"AsyncProxyHandler");
+ try {
+ chain.doFilter(req, res);
+ } catch (Throwable e) {
+ ((HttpServletResponse)res).sendError(HttpServletResponse.SC_BAD_REQUEST,
e.toString());
+ }
+ }
+
+ @Override
+ public void init(FilterConfig arg0) throws ServletException {
+ // TODO Auto-generated method stub
+
+ }
+}
Added:
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/NotFoundHandler.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/NotFoundHandler.java?rev=982588&view=auto
==============================================================================
---
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/NotFoundHandler.java
(added)
+++
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/NotFoundHandler.java
Thu Aug 5 12:51:04 2010
@@ -0,0 +1,39 @@
+package org.apache.nutch.tools.proxy;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletResponse;
+
+import org.mortbay.jetty.Request;
+
+public class NotFoundHandler extends AbstractTestbedHandler {
+
+ @Override
+ public void handle(Request req, HttpServletResponse res, String target,
+ int dispatch) throws IOException, ServletException {
+ // don't pass it down the chain
+ req.setHandled(true);
+ res.addHeader("X-Handled-By", getClass().getSimpleName());
+ addMyHeader(res, "URI", req.getUri().toString());
+ res.sendError(HttpServletResponse.SC_NOT_FOUND, "Not found: " +
+ req.getUri().toString());
+ }
+
+}
Added:
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java?rev=982588&view=auto
==============================================================================
---
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java
(added)
+++
nutch/branches/nutchbase/src/java/org/apache/nutch/tools/proxy/TestbedProxy.java
Thu Aug 5 12:51:04 2010
@@ -0,0 +1,123 @@
+package org.apache.nutch.tools.proxy;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.nutch.util.HadoopFSUtil;
+import org.apache.nutch.util.NutchConfiguration;
+import org.mortbay.jetty.Handler;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.bio.SocketConnector;
+import org.mortbay.jetty.handler.HandlerList;
+import org.mortbay.jetty.servlet.ServletHandler;
+import org.mortbay.proxy.AsyncProxyServlet;
+
+public class TestbedProxy {
+ private static final Log LOG = LogFactory.getLog(TestbedProxy.class);
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length == 0) {
+ System.err.println("TestbedProxy [-port <nnn>] [-forward] [-fake]
[-delay nnn] [-debug]");
+ System.err.println("-port <nnn>\trun the proxy on port <nnn> (special
permissions may be needed for ports < 1024)");
+ System.err.println("-forward\tif specified, requests to all unknown urls
will be passed to");
+ System.err.println("\t\toriginal servers. If false (default) unknown
urls generate 404 Not Found.");
+ System.err.println("-delay\tdelay every response by nnn seconds. If
delay is negative use a random value up to nnn");
+ System.err.println("-fake\tif specified, requests to all unknown urls
will succeed with fake content");
+ System.exit(-1);
+ }
+
+ Configuration conf = NutchConfiguration.create();
+ int port = conf.getInt("segment.proxy.port", 8181);
+ boolean forward = false;
+ boolean fake = false;
+ boolean delay = false;
+ boolean debug = false;
+ int delayVal = 0;
+
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].equals("-port")) {
+ port = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("-forward")) {
+ forward = true;
+ } else if (args[i].equals("-delay")) {
+ delay = true;
+ delayVal = Integer.parseInt(args[++i]);
+ } else if (args[i].equals("-fake")) {
+ fake = true;
+ } else if (args[i].equals("-debug")) {
+ debug = true;
+ } else {
+ LOG.fatal("Unknown argument: " + args[i]);
+ System.exit(-1);
+ }
+ }
+
+ // Create the server
+ Server server = new Server();
+ SocketConnector connector = new SocketConnector();
+ connector.setPort(port);
+ connector.setResolveNames(false);
+ server.addConnector(connector);
+
+ // create a list of handlers
+ HandlerList list = new HandlerList();
+ server.addHandler(list);
+
+ if (debug) {
+ LOG.info("* Added debug handler.");
+ list.addHandler(new LogDebugHandler());
+ }
+
+ if (delay) {
+ LOG.info("* Added delay handler: " + (delayVal < 0 ? "random delay up to
" + (-delayVal) : "constant delay of " + delayVal));
+ list.addHandler(new DelayHandler(delayVal));
+ }
+
+ // XXX alternatively, we can add the DispatchHandler as the first one,
+ // XXX to activate handler plugins and redirect requests to appropriate
+ // XXX handlers ... Here we always load these handlers
+
+ if (forward) {
+ LOG.info("* Adding forwarding proxy for all unknown urls ...");
+ ServletHandler servlets = new ServletHandler();
+ servlets.addServletWithMapping(AsyncProxyServlet.class, "/*");
+ servlets.addFilterWithMapping(LogDebugHandler.class, "/*", Handler.ALL);
+ list.addHandler(servlets);
+ }
+ if (fake) {
+ LOG.info("* Added fake handler for remaining URLs.");
+ list.addHandler(new FakeHandler());
+ }
+ list.addHandler(new NotFoundHandler());
+ // Start the http server
+ server.start();
+ server.join();
+ }
+}
