Hi,
I am going to change crawler class in a way that it can crawl incrementally
based on the number of web pages. Suppose the sum of all pages for 2 depth
crawling is around 5000 pages. Right now this class runs
generate-fetch-update for all pages and after finishing it will send them
to solr for indexing. I want to change this class in a way that it can
break this 5000 pages to 10 different generate-fetch-update cycle. Is that
possible with nutch? If yes how can I do that?
Crawler source:
public class Crawler extends Configured implements Tool {
public static final Logger LOG = LoggerFactory.getLogger(Crawler.class);
private static String getDate() {
return new SimpleDateFormat("yyyyMMddHHmmss").format(new Date(System
.currentTimeMillis()));
}
/*
* Perform complete crawling and indexing (to Solr) given a set of root urls
* and the -solr parameter respectively. More information and Usage
* parameters can be found below.
*/
public static void main(String args[]) throws Exception {
Configuration conf = NutchConfiguration.create();
int res = ToolRunner.run(conf, new Crawler(), args);
System.exit(res);
}
@Override
public int run(String[] args) throws Exception {
if (args.length < 1) {
System.out
.println("Usage: Crawl <urlDir> -solr <solrURL> [-dir d] [-threads n]
[-depth i] [-topN N]");
return -1;
}
Path rootUrlDir = null;
Path dir = new Path("crawl-" + getDate());
int threads = getConf().getInt("fetcher.threads.fetch", 10);
int depth = 5;
long topN = Long.MAX_VALUE;
String solrUrl = null;
for (int i = 0; i < args.length; i++) {
if ("-dir".equals(args[i])) {
dir = new Path(args[i + 1]);
i++;
} else if ("-threads".equals(args[i])) {
threads = Integer.parseInt(args[i + 1]);
i++;
} else if ("-depth".equals(args[i])) {
depth = Integer.parseInt(args[i + 1]);
i++;
} else if ("-topN".equals(args[i])) {
topN = Integer.parseInt(args[i + 1]);
i++;
} else if ("-solr".equals(args[i])) {
solrUrl = args[i + 1];
i++;
} else if (args[i] != null) {
rootUrlDir = new Path(args[i]);
}
}
JobConf job = new NutchJob(getConf());
if (solrUrl == null) {
LOG.warn("solrUrl is not set, indexing will be skipped...");
} else {
// for simplicity assume that SOLR is used
// and pass its URL via conf
getConf().set("solr.server.url", solrUrl);
}
FileSystem fs = FileSystem.get(job);
if (LOG.isInfoEnabled()) {
LOG.info("crawl started in: " + dir);
LOG.info("rootUrlDir = " + rootUrlDir);
LOG.info("threads = " + threads);
LOG.info("depth = " + depth);
LOG.info("solrUrl=" + solrUrl);
if (topN != Long.MAX_VALUE)
LOG.info("topN = " + topN);
}
Path crawlDb = new Path(dir + "/crawldb");
Path linkDb = new Path(dir + "/linkdb");
Path segments = new Path(dir + "/segments");
//Path tmpDir = job.getLocalPath("crawl" + Path.SEPARATOR + getDate());
Injector injector = new Injector(getConf());
Generator generator = new Generator(getConf());
Fetcher fetcher = new Fetcher(getConf());
ParseSegment parseSegment = new ParseSegment(getConf());
CrawlDb crawlDbTool = new CrawlDb(getConf());
LinkDb linkDbTool = new LinkDb(getConf());
// initialize crawlDb
injector.inject(crawlDb, rootUrlDir);
int i;
for (i = 0; i < depth; i++) { // generate new segment
Path[] segs = generator.generate(crawlDb, segments, -1, topN,
System.currentTimeMillis());
if (segs == null) {
LOG.info("Stopping at depth=" + i + " - no more URLs to fetch.");
break;
}
fetcher.fetch(segs[0], threads); // fetch it
if (!Fetcher.isParsing(job)) {
parseSegment.parse(segs[0]); // parse it, if needed
}
crawlDbTool.update(crawlDb, segs, true, true); // update crawldb
}
if (i > 0) {
linkDbTool.invert(linkDb, segments, true, true, false); // invert
// links
// dedup should be added
if (solrUrl != null) {
// index
FileStatus[] fstats = fs.listStatus(segments,
HadoopFSUtil.getPassDirectoriesFilter(fs));
IndexingJob indexer = new IndexingJob(getConf());
boolean noCommit = false;
indexer.index(crawlDb, linkDb,
Arrays.asList(HadoopFSUtil.getPaths(fstats)), noCommit);
}
// merge should be added
// clean should be added
} else {
LOG.warn("No URLs to fetch - check your seed list and URL filters.");
}
if (LOG.isInfoEnabled()) {
LOG.info("crawl finished: " + dir);
}
return 0;
}
}
Best regards.
--
A.Nazemian
