Author: markus
Date: Tue Nov 15 11:56:30 2011
New Revision: 1202143
URL: http://svn.apache.org/viewvc?rev=1202143&view=rev
Log:
NUTCH-1090 InvertLinks should inform when ignoring internal links
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1202143&r1=1202142&r2=1202143&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue Nov 15 11:56:30 2011
@@ -1,5 +1,7 @@
Nutch Change Log
+* NUTCH-1090 InvertLinks should inform when ignoring internal links (Marek
Backmann via markus)
+
* NUTCH-1174 Outlinks are not properly normalized (markus)
* NUTCH-1203 ParseSegment to show number of milliseconds per parse (markus)
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java?rev=1202143&r1=1202142&r2=1202143&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java Tue Nov 15 11:56:30
2011
@@ -47,6 +47,8 @@ public class LinkDb extends Configured i
public static final Logger LOG = LoggerFactory.getLogger(LinkDb.class);
+ public static final String IGNORE_INTERNAL_LINKS =
"db.ignore.internal.links";
+
public static final String CURRENT_NAME = "current";
public static final String LOCK_NAME = ".locked";
@@ -63,7 +65,7 @@ public class LinkDb extends Configured i
public void configure(JobConf job) {
maxAnchorLength = job.getInt("db.max.anchor.length", 100);
- ignoreInternalLinks = job.getBoolean("db.ignore.internal.links", true);
+ ignoreInternalLinks = job.getBoolean(IGNORE_INTERNAL_LINKS, true);
if (job.getBoolean(LinkDbFilter.URL_FILTERING, false)) {
urlFilters = new URLFilters(job);
}
@@ -150,6 +152,7 @@ public class LinkDb extends Configured i
}
public void invert(Path linkDb, Path[] segments, boolean normalize, boolean
filter, boolean force) throws IOException {
+ JobConf job = LinkDb.createJob(getConf(), linkDb, normalize, filter);
Path lock = new Path(linkDb, LOCK_NAME);
FileSystem fs = FileSystem.get(getConf());
LockUtil.createLockFile(fs, lock, force);
@@ -162,9 +165,11 @@ public class LinkDb extends Configured i
LOG.info("LinkDb: linkdb: " + linkDb);
LOG.info("LinkDb: URL normalize: " + normalize);
LOG.info("LinkDb: URL filter: " + filter);
+ if (job.getBoolean(IGNORE_INTERNAL_LINKS, true)) {
+ LOG.info("LinkDb: internal links will be ignored.");
+ }
}
- JobConf job = LinkDb.createJob(getConf(), linkDb, normalize, filter);
for (int i = 0; i < segments.length; i++) {
if (LOG.isInfoEnabled()) {
LOG.info("LinkDb: adding segment: " + segments[i]);