Author: ab
Date: Wed Jul 21 17:48:20 2010
New Revision: 966330
URL: http://svn.apache.org/viewvc?rev=966330&view=rev
Log:
Make some classes easier to test. Fix unit some unit tests.
Added:
nutch/branches/nutchbase/src/test/org/apache/nutch/fetcher/TestFetcher.java
(with props)
nutch/branches/nutchbase/src/test/org/apache/nutch/util/AbstractNutchTest.java
(with props)
nutch/branches/nutchbase/src/test/org/apache/nutch/util/CrawlTestUtil.java
(with props)
Removed:
nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
Modified:
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/GeneratorJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/InjectorJob.java
nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/FetcherJob.java
nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestGenerator.java
nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestInjector.java
Modified:
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/GeneratorJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/GeneratorJob.java?rev=966330&r1=966329&r2=966330&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/GeneratorJob.java
(original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/GeneratorJob.java
Wed Jul 21 17:48:20 2010
@@ -9,6 +9,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
@@ -32,16 +33,13 @@ public class GeneratorJob extends Config
public static final String GENERATOR_COUNT_MODE = "generate.count.mode";
public static final String GENERATOR_COUNT_VALUE_DOMAIN = "domain";
public static final String GENERATOR_COUNT_VALUE_HOST = "host";
+ public static final String GENERATOR_COUNT_VALUE_IP = "ip";
public static final String GENERATOR_TOP_N = "generate.topN";
public static final String GENERATOR_CUR_TIME = "generate.curTime";
public static final String GENERATOR_DELAY = "crawl.gen.delay";
public static final String GENERATOR_RANDOM_SEED = "generate.partition.seed";
public static final String CRAWL_ID = "generate.crawl.id";
- // deprecated parameters
- public static final String GENERATE_MAX_PER_HOST_BY_IP =
"generate.max.per.host.by.ip";
- public static final String GENERATE_MAX_PER_HOST = "generate.max.per.host";
-
private static final Set<WebPage.Field> FIELDS = new
HashSet<WebPage.Field>();
static {
@@ -115,13 +113,21 @@ public class GeneratorJob extends Config
new SelectorEntryComparator());
}
+ public GeneratorJob() {
+
+ }
+
+ public GeneratorJob(Configuration conf) {
+ setConf(conf);
+ }
+
/**
* Mark URLs ready for fetching.
* @throws ClassNotFoundException
* @throws InterruptedException
* */
public String generate(long topN, long curTime, boolean filter, boolean norm)
- throws Exception {
+ throws Exception {
LOG.info("GeneratorJob: Selecting best-scoring urls due for fetch.");
LOG.info("GeneratorJob: starting");
@@ -140,6 +146,16 @@ public class GeneratorJob extends Config
getConf().set(CRAWL_ID, crawlId);
getConf().setLong(Nutch.GENERATE_TIME_KEY, System.currentTimeMillis());
getConf().setBoolean(GENERATOR_NORMALISE, norm);
+ String mode = getConf().get(GENERATOR_COUNT_MODE,
GENERATOR_COUNT_VALUE_HOST);
+ if (GENERATOR_COUNT_VALUE_HOST.equalsIgnoreCase(mode)) {
+ getConf().set(URLPartitioner.PARTITION_MODE_KEY,
URLPartitioner.PARTITION_MODE_HOST);
+ } else if (GENERATOR_COUNT_VALUE_DOMAIN.equalsIgnoreCase(mode)) {
+ getConf().set(URLPartitioner.PARTITION_MODE_KEY,
URLPartitioner.PARTITION_MODE_DOMAIN);
+ } else {
+ LOG.warn("Unknown generator.max.count mode '" + mode + "', using mode="
+ GENERATOR_COUNT_VALUE_HOST);
+ getConf().set(GENERATOR_COUNT_MODE, GENERATOR_COUNT_VALUE_HOST);
+ getConf().set(URLPartitioner.PARTITION_MODE_KEY,
URLPartitioner.PARTITION_MODE_HOST);
+ }
Job job = new NutchJob(getConf(), "generate: " + crawlId);
StorageUtils.initMapperJob(job, FIELDS, SelectorEntry.class, WebPage.class,
Modified:
nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/InjectorJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/InjectorJob.java?rev=966330&r1=966329&r2=966330&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/InjectorJob.java
(original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/crawl/InjectorJob.java
Wed Jul 21 17:48:20 2010
@@ -169,6 +169,14 @@ public class InjectorJob extends GoraMap
}
}
+ public InjectorJob() {
+
+ }
+
+ public InjectorJob(Configuration conf) {
+ setConf(conf);
+ }
+
@Override
public Configuration getConf() {
return conf;
@@ -202,7 +210,7 @@ public class InjectorJob extends GoraMap
context.write(key, row);
}
- void inject(Path urlDir) throws Exception {
+ public void inject(Path urlDir) throws Exception {
LOG.info("InjectorJob: starting");
LOG.info("InjectorJob: urlDir: " + urlDir);
Modified:
nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/FetcherJob.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/FetcherJob.java?rev=966330&r1=966329&r2=966330&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/FetcherJob.java
(original)
+++ nutch/branches/nutchbase/src/java/org/apache/nutch/fetcher/FetcherJob.java
Wed Jul 21 17:48:20 2010
@@ -38,6 +38,10 @@ public class FetcherJob implements Tool
public static final int PERM_REFRESH_TIME = 5;
public static final Utf8 REDIRECT_DISCOVERED = new Utf8("___rdrdsc__");
+
+ public static final String RESUME_KEY = "fetcher.job.resume";
+ public static final String PARSE_KEY = "fetcher.parse";
+ public static final String THREADS_KEY = "fetcher.threads.fetch";
private static final Collection<WebPage.Field> FIELDS = new
HashSet<WebPage.Field>();
@@ -103,6 +107,14 @@ public class FetcherJob implements Tool
public static final Log LOG = LogFactory.getLog(FetcherJob.class);
private Configuration conf;
+
+ public FetcherJob() {
+
+ }
+
+ public FetcherJob(Configuration conf) {
+ setConf(conf);
+ }
@Override
public Configuration getConf() {
@@ -116,7 +128,7 @@ public class FetcherJob implements Tool
public Collection<WebPage.Field> getFields(Job job) {
Collection<WebPage.Field> fields = new HashSet<WebPage.Field>(FIELDS);
- if (job.getConfiguration().getBoolean("fetcher.parse", true)) {
+ if (job.getConfiguration().getBoolean(PARSE_KEY, true)) {
ParserJob parserJob = new ParserJob();
fields.addAll(parserJob.getFields(job));
}
@@ -126,18 +138,18 @@ public class FetcherJob implements Tool
return fields;
}
- private void fetch(int threads, String crawlId, boolean shouldContinue,
boolean isParsing)
- throws Exception {
+ public void fetch(int threads, String crawlId, boolean shouldResume,
boolean isParsing)
+ throws Exception {
LOG.info("FetcherJob: starting");
checkConfiguration();
if (threads > 0) {
- getConf().setInt("fetcher.threads.fetch", threads);
+ getConf().setInt(THREADS_KEY, threads);
}
getConf().set(GeneratorJob.CRAWL_ID, crawlId);
- getConf().setBoolean("fetcher.parse", isParsing);
- getConf().setBoolean("job.continue", shouldContinue);
+ getConf().setBoolean(PARSE_KEY, isParsing);
+ getConf().setBoolean(RESUME_KEY, shouldResume);
// set the actual time for the timelimit relative
// to the beginning of the whole job and not of a specific task
@@ -149,9 +161,9 @@ public class FetcherJob implements Tool
}
LOG.info("FetcherJob : timelimit set for : " + timelimit);
- LOG.info("FetcherJob: threads: " +
getConf().getInt("fetcher.threads.fetch", 10));
- LOG.info("FetcherJob: parsing: " + getConf().getBoolean("fetcher.parse",
true));
- LOG.info("FetcherJob: continuing: " + getConf().getBoolean("job.continue",
false));
+ LOG.info("FetcherJob: threads: " + getConf().getInt(THREADS_KEY, 10));
+ LOG.info("FetcherJob: parsing: " + getConf().getBoolean(PARSE_KEY, true));
+ LOG.info("FetcherJob: resuming: " + getConf().getBoolean(RESUME_KEY,
false));
if (crawlId.equals(Nutch.ALL_CRAWL_ID_STR)) {
LOG.info("FetcherJob: fetching all");
} else {
@@ -170,7 +182,7 @@ public class FetcherJob implements Tool
LOG.info("FetcherJob: done");
}
- private void checkConfiguration() {
+ void checkConfiguration() {
// ensure that a value has been set for the agent name and that that
// agent name is the first value in the agents we advertise for robot
@@ -206,34 +218,34 @@ public class FetcherJob implements Tool
@Override
public int run(String[] args) throws Exception {
int threads = -1;
- boolean shouldContinue = false;
+ boolean shouldResume = false;
boolean isParsing = true;
String crawlId;
- String usage = "Usage: FetcherJob (<crawl id> | -all) [-threads N]
[-noParsing] [-continue]";
+ String usage = "Usage: FetcherJob (<crawl id> | -all) [-threads N]
[-noParsing] [-resume]";
if (args.length == 0) {
System.err.println(usage);
- return 1;
+ return -1;
}
crawlId = args[0];
- if (crawlId.equals("-threads") || crawlId.equals("-continue") ||
crawlId.equals("-noParsing")) {
+ if (crawlId.equals("-threads") || crawlId.equals("-resume") ||
crawlId.equals("-noParsing")) {
System.err.println(usage);
- return 1;
+ return -1;
}
for (int i = 1; i < args.length; i++) {
if ("-threads".equals(args[i])) {
// found -threads option
threads = Integer.parseInt(args[++i]);
- } else if ("-continue".equals(args[i])) {
- shouldContinue = true;
+ } else if ("-resume".equals(args[i])) {
+ shouldResume = true;
} else if ("-noParsing".equals(args[i])) {
isParsing = false;
}
}
- fetch(threads, crawlId, shouldContinue, isParsing); // run the Fetcher
+ fetch(threads, crawlId, shouldResume, isParsing); // run the Fetcher
return 0;
}
Modified:
nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestGenerator.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestGenerator.java?rev=966330&r1=966329&r2=966330&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestGenerator.java
(original)
+++ nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestGenerator.java
Wed Jul 21 17:48:20 2010
@@ -20,20 +20,13 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
-
-import junit.framework.TestCase;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.AbstractNutchTest;
import org.apache.nutch.util.TableUtil;
-import org.gora.query.Query;
-import org.gora.query.Result;
-import org.gora.sql.store.SqlStore;
-import org.gora.store.DataStore;
-import org.gora.store.DataStoreFactory;
/**
* Basic generator test. 1. Insert entries in webtable 2. Generates entries to
@@ -44,32 +37,14 @@ import org.gora.store.DataStoreFactory;
* @param <URLWebPage>
*
*/
-public class TestGenerator extends TestCase {
+public class TestGenerator extends AbstractNutchTest {
public static final Log LOG = LogFactory.getLog(TestGenerator.class);
-
- Configuration conf;
-
- private DataStore<String, WebPage> webPageStore;
-
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- conf = CrawlDBTestUtil.createConfiguration();
- // using hsqldb in memory
-
DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.driver","org.hsqldb.jdbcDriver");
-
DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.url","jdbc:hsqldb:mem:.");
- DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.user","sa");
- DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.password","");
- webPageStore = DataStoreFactory.getDataStore(SqlStore.class,
- String.class, WebPage.class);
- }
-
- @Override
- protected void tearDown() throws Exception {
- webPageStore.close();
- super.tearDown();
- }
+
+ private static String[] FIELDS = new String[] {
+ WebPage.Field.MARKERS.getName(),
+ WebPage.Field.SCORE.getName()
+ };
/**
* Test that generator generates fetchlish ordered by score (desc).
@@ -89,10 +64,11 @@ public class TestGenerator extends TestC
for (URLWebPage uwp : list) {
webPageStore.put(TableUtil.reverseUrl(uwp.getUrl()), uwp.getDatum());
}
+ webPageStore.flush();
generateFetchlist(NUM_RESULTS, conf, false);
- ArrayList<URLWebPage> l = readContents();
+ ArrayList<URLWebPage> l = readContents(webPageStore, Mark.GENERATE_MARK,
FIELDS);
// sort urls by score desc
Collections.sort(l, new ScoreComparator());
@@ -144,77 +120,83 @@ public class TestGenerator extends TestC
for (URLWebPage uwp : list) {
webPageStore.put(TableUtil.reverseUrl(uwp.getUrl()), uwp.getDatum());
}
+ webPageStore.flush();
Configuration myConfiguration = new Configuration(conf);
- myConfiguration.setInt(GeneratorJob.GENERATE_MAX_PER_HOST, 1);
+ myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 1);
+ myConfiguration.set(GeneratorJob.GENERATOR_COUNT_MODE,
GeneratorJob.GENERATOR_COUNT_VALUE_HOST);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
- ArrayList<URLWebPage> fetchList = readContents();
+ ArrayList<URLWebPage> fetchList = readContents(webPageStore,
Mark.GENERATE_MARK, FIELDS);
// verify we got right amount of records
assertEquals(1, fetchList.size());
myConfiguration = new Configuration(conf);
- myConfiguration.setInt(GeneratorJob.GENERATE_MAX_PER_HOST, 2);
+ myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 2);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
- fetchList = readContents();
+ fetchList = readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
// verify we got right amount of records
assertEquals(2, fetchList.size());
myConfiguration = new Configuration(conf);
- myConfiguration.setInt(GeneratorJob.GENERATE_MAX_PER_HOST, 3);
+ myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 3);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
- fetchList = readContents();
+ fetchList = readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
// verify we got right amount of records
assertEquals(3, fetchList.size());
}
/**
- * Test that generator obeys the property "generate.max.per.host" and
- * "generate.max.per.host.by.ip".
+ * Test that generator obeys the property "generator.max.count" and
+ * "generator.count.value=domain".
*
* @throws Exception
*/
- public void testGenerateHostIPLimit() throws Exception {
+ public void testGenerateDomainLimit() throws Exception {
ArrayList<URLWebPage> list = new ArrayList<URLWebPage>();
- list.add(createURLWebPage("http://www.example.com/index.html";, 1, 1));
- list.add(createURLWebPage("http://www.example.net/index.html";, 1, 1));
- list.add(createURLWebPage("http://www.example.org/index.html";, 1, 1));
+ list.add(createURLWebPage("http://one.example.com/index.html";, 1, 1));
+ list.add(createURLWebPage("http://one.example.com/index1.html";, 1, 1));
+ list.add(createURLWebPage("http://two.example.com/index.html";, 1, 1));
+ list.add(createURLWebPage("http://two.example.com/index1.html";, 1, 1));
+ list.add(createURLWebPage("http://three.example.com/index.html";, 1, 1));
+ list.add(createURLWebPage("http://three.example.com/index1.html";, 1, 1));
for (URLWebPage uwp : list) {
webPageStore.put(TableUtil.reverseUrl(uwp.getUrl()), uwp.getDatum());
}
-
+ webPageStore.flush();
+
Configuration myConfiguration = new Configuration(conf);
- myConfiguration.setInt(GeneratorJob.GENERATE_MAX_PER_HOST, 1);
- myConfiguration.setBoolean(GeneratorJob.GENERATE_MAX_PER_HOST_BY_IP, true);
+ myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 1);
+ myConfiguration.set(GeneratorJob.GENERATOR_COUNT_MODE,
GeneratorJob.GENERATOR_COUNT_VALUE_DOMAIN);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
- ArrayList<URLWebPage> fetchList = readContents();
+ ArrayList<URLWebPage> fetchList = readContents(webPageStore,
Mark.GENERATE_MARK, FIELDS);
// verify we got right amount of records
assertEquals(1, fetchList.size());
myConfiguration = new Configuration(myConfiguration);
- myConfiguration.setInt(GeneratorJob.GENERATE_MAX_PER_HOST, 2);
+ myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 2);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
- fetchList = readContents();
+ fetchList = readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
// verify we got right amount of records
assertEquals(2, fetchList.size());
myConfiguration = new Configuration(myConfiguration);
- myConfiguration.setInt(GeneratorJob.GENERATE_MAX_PER_HOST, 3);
+ myConfiguration.setInt(GeneratorJob.GENERATOR_MAX_COUNT, 3);
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
- fetchList = readContents();
+ fetchList = readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
// verify we got right amount of records
assertEquals(3, fetchList.size());
@@ -237,19 +219,20 @@ public class TestGenerator extends TestC
for (URLWebPage uwp : list) {
webPageStore.put(TableUtil.reverseUrl(uwp.getUrl()), uwp.getDatum());
}
+ webPageStore.flush();
Configuration myConfiguration = new Configuration(conf);
myConfiguration.set("urlfilter.suffix.file", "filter-all.txt");
generateFetchlist(Integer.MAX_VALUE, myConfiguration, true);
- ArrayList<URLWebPage> fetchList = readContents();
+ ArrayList<URLWebPage> fetchList = readContents(webPageStore,
Mark.GENERATE_MARK, FIELDS);
assertEquals(0, fetchList.size());
generateFetchlist(Integer.MAX_VALUE, myConfiguration, false);
- fetchList = readContents();
+ fetchList = readContents(webPageStore, Mark.GENERATE_MARK, FIELDS);
// verify nothing got filtered
assertEquals(list.size(), fetchList.size());
@@ -257,38 +240,6 @@ public class TestGenerator extends TestC
}
/**
- * Read entries marked as fetchable
- *
- * @return Generated {...@link URLWebPage} objects
- * @throws IOException
- */
- private ArrayList<URLWebPage> readContents() throws IOException {
- ArrayList<URLWebPage> l = new ArrayList<URLWebPage>();
-
- Query<String, WebPage> query = webPageStore.newQuery();
- query.setFields(WebPage.Field.MARKERS.getName());
-
- Result<String, WebPage> results = webPageStore.execute(query);
-
- while (results.next()) {
- WebPage page = results.get();
- String url = results.getKey();
- LOG.info("FOUND IN TABLE :" + url);
-
- if (page == null)
- continue;
-
- if (Mark.GENERATE_MARK.checkMark(page) == null)
- continue;
-
- // it has been marked as generated so it is ready for the fetch
- l.add(new URLWebPage(TableUtil.unreverseUrl(url), page));
- }
-
- return l;
- }
-
- /**
* Generate Fetchlist.
*
* @param numResults
@@ -303,7 +254,7 @@ public class TestGenerator extends TestC
// generate segment
GeneratorJob g = new GeneratorJob();
g.setConf(config);
- String crawlId = g.generate(numResults, Long.MAX_VALUE, filter, false);
+ String crawlId = g.generate(numResults, System.currentTimeMillis(),
filter, false);
if (crawlId == null)
throw new RuntimeException("Generator failed");
}
Modified:
nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestInjector.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestInjector.java?rev=966330&r1=966329&r2=966330&view=diff
==============================================================================
--- nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestInjector.java
(original)
+++ nutch/branches/nutchbase/src/test/org/apache/nutch/crawl/TestInjector.java
Wed Jul 21 17:48:20 2010
@@ -27,7 +27,10 @@ import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.AbstractNutchTest;
+import org.apache.nutch.util.CrawlTestUtil;
import org.apache.nutch.util.TableUtil;
import org.gora.query.Query;
import org.gora.query.Result;
@@ -44,37 +47,14 @@ import org.junit.Before;
*
* @author nutch-dev <nutch-dev at lucene.apache.org>
*/
-public class TestInjector extends TestCase {
-
- private Configuration conf;
- private FileSystem fs;
- final static Path testdir = new Path("build/test/inject-test");
- private DataStore<String, WebPage> webPageStore;
+public class TestInjector extends AbstractNutchTest {
Path urlPath;
@Before
@Override
public void setUp() throws Exception {
super.setUp();
- conf = CrawlDBTestUtil.createConfiguration();
urlPath = new Path(testdir, "urls");
- fs = FileSystem.get(conf);
- if (fs.exists(urlPath))
- fs.delete(urlPath, false);
- // using hsqldb in memory
-
DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.driver","org.hsqldb.jdbcDriver");
-
DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.url","jdbc:hsqldb:mem:.");
- DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.user","sa");
- DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.password","");
- webPageStore = DataStoreFactory.getDataStore(SqlStore.class,
- String.class, WebPage.class);
- }
-
- @Override
- public void tearDown() throws Exception {
- webPageStore.close();
- super.tearDown();
- fs.delete(testdir, true);
}
public void testInject() throws Exception {
@@ -83,14 +63,14 @@ public class TestInjector extends TestCa
urls.add("http://zzz.com/"; + i + ".html\tnutch.score=" + i
+ "\tcustom.attribute=" + i);
}
- CrawlDBTestUtil.generateSeedList(fs, urlPath, urls);
+ CrawlTestUtil.generateSeedList(fs, urlPath, urls);
InjectorJob injector = new InjectorJob();
injector.setConf(conf);
injector.inject(urlPath);
// verify results
- List<String> read = readCrawldb();
+ List<String> read = readDb();
Collections.sort(read);
Collections.sort(urls);
@@ -102,15 +82,18 @@ public class TestInjector extends TestCa
// inject more urls
ArrayList<String> urls2 = new ArrayList<String>();
+ ArrayList<String> urlsCheck = new ArrayList<String>();
for (int i = 0; i < 100; i++) {
- urls2.add("http://xxx.com/"; + i + ".html");
+ String u = "http://xxx.com/"; + i + ".html";
+ urls2.add(u);
+ urlsCheck.add(u + "\tnutch.score=1");
}
- CrawlDBTestUtil.generateSeedList(fs, urlPath, urls2);
+ CrawlTestUtil.generateSeedList(fs, urlPath, urls2);
injector.inject(urlPath);
- urls.addAll(urls2);
+ urls.addAll(urlsCheck);
// verify results
- read = readCrawldb();
+ read = readDb();
Collections.sort(read);
Collections.sort(urls);
@@ -121,32 +104,26 @@ public class TestInjector extends TestCa
assertTrue(urls.containsAll(read));
}
-
- /**
- * Read from a Gora datastore + make sure we get the score and custom
metadata
- *
- * @throws ClassNotFoundException
- **/
- private List<String> readCrawldb() throws Exception {
+
+ private static final String[] fields = new String[] {
+ WebPage.Field.MARKERS.getName(),
+ WebPage.Field.METADATA.getName(),
+ WebPage.Field.SCORE.getName()
+ };
+
+ private List<String> readDb() throws Exception {
+ List<URLWebPage> pages = readContents(webPageStore, null, fields);
ArrayList<String> read = new ArrayList<String>();
-
- Query<String, WebPage> query = webPageStore.newQuery();
- Result<String, WebPage> result = webPageStore.execute(query);
-
- while (result.next()) {
- String skey = result.getKey();
- WebPage page = result.get();
- float fscore = page.getScore();
- String representation = TableUtil.unreverseUrl(skey);
+ for (URLWebPage up : pages) {
+ WebPage page = up.getDatum();
+ String representation = up.getUrl();
+ representation += "\tnutch.score=" + (int)page.getScore();
ByteBuffer bb = page.getFromMetadata(new Utf8("custom.attribute"));
if (bb != null) {
- representation += "\tnutch.score=" + (int) fscore;
representation += "\tcustom.attribute=" +
ByteUtils.toString(bb.array());
}
read.add(representation);
}
- result.close();
return read;
}
-
}
Added:
nutch/branches/nutchbase/src/test/org/apache/nutch/fetcher/TestFetcher.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/test/org/apache/nutch/fetcher/TestFetcher.java?rev=966330&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/test/org/apache/nutch/fetcher/TestFetcher.java
(added)
+++ nutch/branches/nutchbase/src/test/org/apache/nutch/fetcher/TestFetcher.java
Wed Jul 21 17:48:20 2010
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.fetcher;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.nutch.crawl.GeneratorJob;
+import org.apache.nutch.crawl.InjectorJob;
+import org.apache.nutch.crawl.URLWebPage;
+import org.apache.nutch.storage.Mark;
+import org.apache.nutch.util.AbstractNutchTest;
+import org.apache.nutch.util.CrawlTestUtil;
+import org.mortbay.jetty.Server;
+
+/**
+ * Basic fetcher test
+ * 1. generate seedlist
+ * 2. inject
+ * 3. generate
+ * 3. fetch
+ * 4. Verify contents
+ * @author nutch-dev <nutch-dev at lucene.apache.org>
+ *
+ */
+public class TestFetcher extends AbstractNutchTest {
+
+ final static Path testdir=new Path("build/test/fetch-test");
+ Path urlPath;
+ Server server;
+
+ public void setUp() throws Exception{
+ super.setUp();
+ urlPath = new Path(testdir, "urls");
+ server = CrawlTestUtil.getServer(conf.getInt("content.server.port",50000),
"build/test/data/fetch-test-site");
+ server.start();
+ }
+
+ public void tearDown() throws Exception{
+ server.stop();
+ fs.delete(testdir, true);
+ }
+
+ public void testFetch() throws Exception {
+
+ //generate seedlist
+ ArrayList<String> urls = new ArrayList<String>();
+
+ addUrl(urls,"index.html");
+ addUrl(urls,"pagea.html");
+ addUrl(urls,"pageb.html");
+ addUrl(urls,"dup_of_pagea.html");
+ addUrl(urls,"nested_spider_trap.html");
+ addUrl(urls,"exception.html");
+
+ CrawlTestUtil.generateSeedList(fs, urlPath, urls);
+
+ //inject
+ InjectorJob injector = new InjectorJob(conf);
+ injector.inject(urlPath);
+
+ //generate
+ long time = System.currentTimeMillis();
+ GeneratorJob g = new GeneratorJob(conf);
+ String crawlId = g.generate(Long.MAX_VALUE, time, false, false);
+
+ //fetch
+ time = System.currentTimeMillis();
+ conf.setBoolean(FetcherJob.PARSE_KEY, true);
+ FetcherJob fetcher = new FetcherJob(conf);
+ fetcher.fetch(1, crawlId, false, true);
+
+ time = System.currentTimeMillis() - time;
+
+ //verify politeness, time taken should be more than (num_of_pages +1)*delay
+ int minimumTime = (int) ((urls.size() + 1) * 1000 *
+ conf.getFloat("fetcher.server.delay", 5));
+ assertTrue(time > minimumTime);
+
+ List<URLWebPage> pages = readContents(webPageStore, Mark.FETCH_MARK, null);
+ assertEquals(urls.size(), pages.size());
+ List<String> handledurls = new ArrayList<String>();
+ for (URLWebPage up : pages) {
+ ByteBuffer bb = up.getDatum().getContent();
+ if (bb == null) {
+ continue;
+ }
+ String content = new String(bb.array());
+ if (content.indexOf("Nutch fetcher test page")!=-1) {
+ handledurls.add(up.getUrl());
+ }
+ }
+ Collections.sort(urls);
+ Collections.sort(handledurls);
+
+ //verify that enough pages were handled
+ assertEquals(urls.size(), handledurls.size());
+
+ //verify that correct pages were handled
+ assertTrue(handledurls.containsAll(urls));
+ assertTrue(urls.containsAll(handledurls));
+ }
+
+ private void addUrl(ArrayList<String> urls, String page) {
+ urls.add("http://127.0.0.1:"; + server.getConnectors()[0].getPort() + "/" +
page);
+ }
+
+ public void testAgentNameCheck() {
+
+ boolean failedNoAgentName = false;
+ conf.set("http.agent.name", "");
+
+ try {
+ conf.setBoolean(FetcherJob.PARSE_KEY, true);
+ FetcherJob fetcher = new FetcherJob(conf);
+ fetcher.checkConfiguration();
+ } catch (IllegalArgumentException iae) {
+ String message = iae.getMessage();
+ failedNoAgentName = message.equals("Fetcher: No agents listed in "
+ + "'http.agent.name' property.");
+ } catch (Exception e) {
+ }
+
+ assertTrue(failedNoAgentName);
+ }
+
+}
Propchange:
nutch/branches/nutchbase/src/test/org/apache/nutch/fetcher/TestFetcher.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
nutch/branches/nutchbase/src/test/org/apache/nutch/util/AbstractNutchTest.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/test/org/apache/nutch/util/AbstractNutchTest.java?rev=966330&view=auto
==============================================================================
---
nutch/branches/nutchbase/src/test/org/apache/nutch/util/AbstractNutchTest.java
(added)
+++
nutch/branches/nutchbase/src/test/org/apache/nutch/util/AbstractNutchTest.java
Wed Jul 21 17:48:20 2010
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.apache.avro.util.Utf8;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.nutch.crawl.URLWebPage;
+import org.apache.nutch.storage.Mark;
+import org.apache.nutch.storage.WebPage;
+import org.apache.nutch.util.TableUtil;
+import org.gora.query.Query;
+import org.gora.query.Result;
+import org.gora.sql.store.SqlStore;
+import org.gora.store.DataStore;
+import org.gora.store.DataStoreFactory;
+import org.gora.util.ByteUtils;
+
+/**
+ * Basic injector test: 1. Creates a text file with urls 2. Injects them into
+ * crawldb 3. Reads crawldb entries and verifies contents 4. Injects more urls
+ * into webdb 5. Reads crawldb entries and verifies contents
+ *
+ * @author nutch-dev <nutch-dev at lucene.apache.org>
+ */
+public class AbstractNutchTest extends TestCase {
+
+ protected Configuration conf;
+ protected FileSystem fs;
+ protected Path testdir = new Path("build/test/inject-test");
+ protected DataStore<String, WebPage> webPageStore;
+ protected boolean persistentDataStore = false;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ conf = CrawlTestUtil.createConfiguration();
+ fs = FileSystem.get(conf);
+ // using hsqldb in memory
+
DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.driver","org.hsqldb.jdbcDriver");
+ // use separate in-memory db-s for tests
+
DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.url","jdbc:hsqldb:mem:"
+ getClass().getName());
+ DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.user","sa");
+ DataStoreFactory.properties.setProperty("gora.sqlstore.jdbc.password","");
+ if (persistentDataStore) {
+ webPageStore = DataStoreFactory.getDataStore(SqlStore.class,
+ String.class, WebPage.class);
+ } else {
+ webPageStore = DataStoreFactory.createDataStore(SqlStore.class,
+ String.class, WebPage.class);
+ }
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ // empty the database after test
+ if (!persistentDataStore) {
+ webPageStore.deleteByQuery(webPageStore.newQuery());
+ webPageStore.flush();
+ webPageStore.close();
+ }
+ super.tearDown();
+ fs.delete(testdir, true);
+ }
+
+ /**
+ * Read entries from a data store
+ *
+ * @return list of matching {...@link URLWebPage} objects
+ * @throws IOException
+ */
+ public static ArrayList<URLWebPage> readContents(DataStore<String,WebPage>
store,
+ Mark requiredMark, String... fields) throws IOException {
+ ArrayList<URLWebPage> l = new ArrayList<URLWebPage>();
+
+ Query<String, WebPage> query = store.newQuery();
+ if (fields != null) {
+ query.setFields(fields);
+ }
+
+ Result<String, WebPage> results = store.execute(query);
+ while (results.next()) {
+ WebPage page = results.get();
+ String url = results.getKey();
+
+ if (page == null)
+ continue;
+
+ if (requiredMark != null && requiredMark.checkMark(page) == null)
+ continue;
+
+ l.add(new URLWebPage(TableUtil.unreverseUrl(url),
(WebPage)page.clone()));
+ }
+
+ return l;
+ }
+}
Propchange:
nutch/branches/nutchbase/src/test/org/apache/nutch/util/AbstractNutchTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
nutch/branches/nutchbase/src/test/org/apache/nutch/util/CrawlTestUtil.java
URL:
http://svn.apache.org/viewvc/nutch/branches/nutchbase/src/test/org/apache/nutch/util/CrawlTestUtil.java?rev=966330&view=auto
==============================================================================
--- nutch/branches/nutchbase/src/test/org/apache/nutch/util/CrawlTestUtil.java
(added)
+++ nutch/branches/nutchbase/src/test/org/apache/nutch/util/CrawlTestUtil.java
Wed Jul 21 17:48:20 2010
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.util;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.mortbay.jetty.Handler;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.handler.DefaultHandler;
+import org.mortbay.jetty.handler.HandlerList;
+import org.mortbay.jetty.handler.ResourceHandler;
+import org.mortbay.jetty.servlet.Context;
+
+import com.sun.net.httpserver.HttpContext;
+
+public class CrawlTestUtil {
+
+ private static final Log LOG = LogFactory.getLog(CrawlTestUtil.class);
+
+ /**
+ * For now we need to manually construct our Configuration, because we need
to
+ * override the default one and it is currently not possible to use
+ * dynamically set values.
+ *
+ * @return
+ * @deprecated Use {...@link #createConfiguration()} instead
+ */
+ public static Configuration create() {
+ return createConfiguration();
+ }
+
+ /**
+ * For now we need to manually construct our Configuration, because we need
to
+ * override the default one and it is currently not possible to use
+ * dynamically set values.
+ *
+ * @return
+ */
+ public static Configuration createConfiguration() {
+ Configuration conf = new Configuration();
+ conf.addResource("nutch-default.xml");
+ conf.addResource("nutch-site.xml");
+ conf.addResource("crawl-tests.xml");
+ return conf;
+ }
+
+ /**
+ * Generate seedlist
+ *
+ * @see TestInjector
+ * @throws IOException
+ */
+ public static void generateSeedList(FileSystem fs, Path urlPath,
+ List<String> contents) throws IOException {
+ FSDataOutputStream out;
+ Path file = new Path(urlPath, "urls.txt");
+ fs.mkdirs(urlPath);
+ out = fs.create(file);
+ Iterator<String> iterator = contents.iterator();
+ while (iterator.hasNext()) {
+ String url = iterator.next();
+ out.writeBytes(url);
+ out.writeBytes("\n");
+ }
+ out.flush();
+ out.close();
+ }
+
+ /**
+ * Creates a new JettyServer with one static root context
+ *
+ * @param port
+ * port to listen to
+ * @param staticContent
+ * folder where static content lives
+ * @throws UnknownHostException
+ */
+ public static Server getServer(int port, String staticContent)
+ throws UnknownHostException {
+ Server webServer = new org.mortbay.jetty.Server(port);
+ ResourceHandler handler = new ResourceHandler();
+ handler.setResourceBase(staticContent);
+ HandlerList handlers = new HandlerList();
+ handlers.setHandlers(new Handler[]{handler, new DefaultHandler()});
+ webServer.setHandler(handlers);
+ return webServer;
+ }
+}
Propchange:
nutch/branches/nutchbase/src/test/org/apache/nutch/util/CrawlTestUtil.java
------------------------------------------------------------------------------
svn:eol-style = native
