Author: markus
Date: Thu Nov 10 14:27:53 2011
New Revision: 1200344

URL: http://svn.apache.org/viewvc?rev=1200344&view=rev
Log:
NUTCH-1153 LinkRank not to log all keys and not to write Hadoop _SUCCESS file

Modified:
    nutch/trunk/CHANGES.txt
    nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java

Modified: nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1200344&r1=1200343&r2=1200344&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu Nov 10 14:27:53 2011
@@ -1,5 +1,7 @@
 Nutch Change Log
 
+* NUTCH-1153 LinkRank not to log all keys and not to write Hadoop _SUCCESS 
file (markus)
+
 Release 1.4 - 11/4/2011
 
 * NUTCH-1195 Add Solr 4x (trunk) example schema (ab)

Modified: nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java?rev=1200344&r1=1200343&r2=1200344&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java 
(original)
+++ nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java Thu 
Nov 10 14:27:53 2011
@@ -105,6 +105,7 @@ public class LinkRank
     counter.setOutputValueClass(LongWritable.class);
     counter.setNumReduceTasks(1);
     counter.setOutputFormat(TextOutputFormat.class);
+    counter.setBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", 
false);
 
     // run the counter job, outputs to a single reduce task and file
     LOG.info("Starting link counter job");
@@ -162,6 +163,7 @@ public class LinkRank
     initializer.setOutputKeyClass(Text.class);
     initializer.setOutputValueClass(Node.class);
     initializer.setOutputFormat(MapFileOutputFormat.class);
+    initializer.setBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", 
false);
 
     // run the initializer
     LOG.info("Starting initialization job");
@@ -213,6 +215,7 @@ public class LinkRank
     inverter.setOutputKeyClass(Text.class);
     inverter.setOutputValueClass(LinkDatum.class);
     inverter.setOutputFormat(SequenceFileOutputFormat.class);
+    inverter.setBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", 
false);
 
     // run the inverter job
     LOG.info("Starting inverter job");
@@ -262,6 +265,7 @@ public class LinkRank
     analyzer.setOutputKeyClass(Text.class);
     analyzer.setOutputValueClass(Node.class);
     analyzer.setOutputFormat(MapFileOutputFormat.class);
+    analyzer.setBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", 
false);
 
     LOG.info("Starting analysis job");
     try {
@@ -537,7 +541,7 @@ public class LinkRank
       float linkRankScore = (1 - this.dampingFactor)
         + (this.dampingFactor * totalInlinkScore);
 
-      LOG.info(url + ": score: " + linkRankScore + " num inlinks: "
+      LOG.debug(url + ": score: " + linkRankScore + " num inlinks: "
         + numInlinks + " iteration: " + itNum);
 
       // store the score in a temporary NodeDb


Reply via email to