Author: markus
Date: Tue Nov 15 11:56:30 2011
New Revision: 1202143

URL: http://svn.apache.org/viewvc?rev=1202143&view=rev
Log:
NUTCH-1090 InvertLinks should inform when ignoring internal links

Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java

Modified: nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1202143&r1=1202142&r2=1202143&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue Nov 15 11:56:30 2011
@@ -1,5 +1,7 @@
 Nutch Change Log
 
+* NUTCH-1090 InvertLinks should inform when ignoring internal links (Marek 
Backmann via markus)
+
 * NUTCH-1174 Outlinks are not properly normalized (markus)
 
 * NUTCH-1203 ParseSegment to show number of milliseconds per parse (markus)

Modified: nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java?rev=1202143&r1=1202142&r2=1202143&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java Tue Nov 15 11:56:30 
2011
@@ -47,6 +47,8 @@ public class LinkDb extends Configured i
 
   public static final Logger LOG = LoggerFactory.getLogger(LinkDb.class);
 
+  public static final String IGNORE_INTERNAL_LINKS = 
"db.ignore.internal.links";
+
   public static final String CURRENT_NAME = "current";
   public static final String LOCK_NAME = ".locked";
 
@@ -63,7 +65,7 @@ public class LinkDb extends Configured i
   
   public void configure(JobConf job) {
     maxAnchorLength = job.getInt("db.max.anchor.length", 100);
-    ignoreInternalLinks = job.getBoolean("db.ignore.internal.links", true);
+    ignoreInternalLinks = job.getBoolean(IGNORE_INTERNAL_LINKS, true);
     if (job.getBoolean(LinkDbFilter.URL_FILTERING, false)) {
       urlFilters = new URLFilters(job);
     }
@@ -150,6 +152,7 @@ public class LinkDb extends Configured i
   }
 
   public void invert(Path linkDb, Path[] segments, boolean normalize, boolean 
filter, boolean force) throws IOException {
+    JobConf job = LinkDb.createJob(getConf(), linkDb, normalize, filter);
     Path lock = new Path(linkDb, LOCK_NAME);
     FileSystem fs = FileSystem.get(getConf());
     LockUtil.createLockFile(fs, lock, force);
@@ -162,9 +165,11 @@ public class LinkDb extends Configured i
       LOG.info("LinkDb: linkdb: " + linkDb);
       LOG.info("LinkDb: URL normalize: " + normalize);
       LOG.info("LinkDb: URL filter: " + filter);
+      if (job.getBoolean(IGNORE_INTERNAL_LINKS, true)) {
+        LOG.info("LinkDb: internal links will be ignored.");
+      }
     }
 
-    JobConf job = LinkDb.createJob(getConf(), linkDb, normalize, filter);
     for (int i = 0; i < segments.length; i++) {
       if (LOG.isInfoEnabled()) {
         LOG.info("LinkDb: adding segment: " + segments[i]);


Reply via email to