Author: ab
Date: Thu Aug 19 11:31:38 2010
New Revision: 987132
URL: http://svn.apache.org/viewvc?rev=987132&view=rev
Log:
Improve ProtocolStatusUtils and move it to a more appropriate place.
Add the display of protocolStatus in readdb -dump.
Hook up WebTableReader to bin/nutch readdb.
Added:
nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java
(with props)
Removed:
nutch/trunk/src/java/org/apache/nutch/storage/ProtocolStatusUtils.java
Modified:
nutch/trunk/src/bin/nutch
nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java
nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java
nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java
nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolOutput.java
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
Modified: nutch/trunk/src/bin/nutch
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/bin/nutch?rev=987132&r1=987131&r2=987132&view=diff
==============================================================================
--- nutch/trunk/src/bin/nutch (original)
+++ nutch/trunk/src/bin/nutch Thu Aug 19 11:31:38 2010
@@ -38,6 +38,7 @@ if [ $# = 0 ]; then
echo " fetch fetch URLs marked during generate"
echo " parse parse URLs marked during fetch"
echo " updatedb update web table after parsing"
+ echo " readdb read/dump records from page database"
echo " solrindex run the solr indexer on parsed segments and linkdb"
echo " solrdedup remove duplicates from solr"
echo " plugin load a plugin and run one of its classes main()"
@@ -180,6 +181,8 @@ elif [ "$COMMAND" = "parse" ] ; then
CLASS=org.apache.nutch.parse.ParserJob
elif [ "$COMMAND" = "updatedb" ] ; then
CLASS=org.apache.nutch.crawl.DbUpdaterJob
+elif [ "$COMMAND" = "readdb" ] ; then
+CLASS=org.apache.nutch.crawl.WebTableReader
elif [ "$COMMAND" = "solrindex" ] ; then
CLASS=org.apache.nutch.indexer.solr.SolrIndexerJob
elif [ "$COMMAND" = "solrdedup" ] ; then
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java?rev=987132&r1=987131&r2=987132&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/WebTableReader.java Thu Aug 19
11:31:38 2010
@@ -28,6 +28,9 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.parse.ParseStatusUtils;
+import org.apache.nutch.protocol.ProtocolStatusUtils;
+import org.apache.nutch.storage.ParseStatus;
+import org.apache.nutch.storage.ProtocolStatus;
import org.apache.nutch.storage.StorageUtils;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.Bytes;
@@ -422,7 +425,9 @@ public class WebTableReader extends Conf
sb.append("prevFetchTime:\t" + page.getPrevFetchTime()).append("\n");
sb.append("retries:\t" + page.getRetriesSinceFetch()).append("\n");
sb.append("modifiedTime:\t" + page.getModifiedTime()).append("\n");
- sb.append("parse_status:\t" +
+ sb.append("protocolStatus:\t" +
+ ProtocolStatusUtils.toString(page.getProtocolStatus())).append("\n");
+ sb.append("parseStatus:\t" +
ParseStatusUtils.toString(page.getParseStatus())).append("\n");
sb.append("title:\t" + page.getTitle()).append("\n");
sb.append("score:\t" + page.getScore()).append("\n");
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java?rev=987132&r1=987131&r2=987132&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherReducer.java Thu Aug
19 11:31:38 2010
@@ -32,10 +32,10 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.protocol.ProtocolFactory;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatusCodes;
+import org.apache.nutch.protocol.ProtocolStatusUtils;
import org.apache.nutch.protocol.RobotRules;
import org.apache.nutch.storage.Mark;
import org.apache.nutch.storage.ProtocolStatus;
-import org.apache.nutch.storage.ProtocolStatusUtils;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.TableUtil;
Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java?rev=987132&r1=987131&r2=987132&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/ParserJob.java Thu Aug 19
11:31:38 2010
@@ -43,6 +43,7 @@ public class ParserJob extends GoraMappe
FIELDS.add(WebPage.Field.CONTENT_TYPE);
FIELDS.add(WebPage.Field.SIGNATURE);
FIELDS.add(WebPage.Field.MARKERS);
+ FIELDS.add(WebPage.Field.PARSE_STATUS);
FIELDS.add(WebPage.Field.OUTLINKS);
FIELDS.add(WebPage.Field.METADATA);
}
Modified: nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolOutput.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolOutput.java?rev=987132&r1=987131&r2=987132&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolOutput.java
(original)
+++ nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolOutput.java Thu Aug
19 11:31:38 2010
@@ -17,7 +17,6 @@
package org.apache.nutch.protocol;
-import org.apache.nutch.storage.ProtocolStatusUtils;
/**
* Simple aggregate to pass from protocol plugins both content and
Added: nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java?rev=987132&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java
(added)
+++ nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java Thu
Aug 19 11:31:38 2010
@@ -0,0 +1,107 @@
+package org.apache.nutch.protocol;
+
+import java.net.URL;
+import java.util.Iterator;
+
+import org.apache.avro.generic.GenericArray;
+import org.apache.avro.util.Utf8;
+import org.apache.nutch.storage.ProtocolStatus;
+import org.apache.nutch.util.TableUtil;
+
+public class ProtocolStatusUtils implements ProtocolStatusCodes {
+ // Useful static instances for status codes that don't usually require any
+ // additional arguments.
+ public static final ProtocolStatus STATUS_SUCCESS = makeStatus(SUCCESS);
+ public static final ProtocolStatus STATUS_FAILED = makeStatus(FAILED);
+ public static final ProtocolStatus STATUS_GONE = makeStatus(GONE);
+ public static final ProtocolStatus STATUS_NOTFOUND = makeStatus(NOTFOUND);
+ public static final ProtocolStatus STATUS_RETRY = makeStatus(RETRY);
+ public static final ProtocolStatus STATUS_ROBOTS_DENIED =
makeStatus(ROBOTS_DENIED);
+ public static final ProtocolStatus STATUS_REDIR_EXCEEDED =
makeStatus(REDIR_EXCEEDED);
+ public static final ProtocolStatus STATUS_NOTFETCHING =
makeStatus(NOTFETCHING);
+ public static final ProtocolStatus STATUS_NOTMODIFIED =
makeStatus(NOTMODIFIED);
+ public static final ProtocolStatus STATUS_WOULDBLOCK =
makeStatus(WOULDBLOCK);
+ public static final ProtocolStatus STATUS_BLOCKED = makeStatus(BLOCKED);
+
+ public static String getName(int code) {
+ if (code == SUCCESS)
+ return "SUCCESS";
+ else if (code == FAILED)
+ return "FAILED";
+ else if (code == PROTO_NOT_FOUND)
+ return "PROTO_NOT_FOUND";
+ else if (code == GONE)
+ return "GONE";
+ else if (code == MOVED)
+ return "MOVED";
+ else if (code == TEMP_MOVED)
+ return "TEMP_MOVED";
+ else if (code == NOTFOUND)
+ return "NOTFOUND";
+ else if (code == RETRY)
+ return "RETRY";
+ else if (code == EXCEPTION)
+ return "EXCEPTION";
+ else if (code == ACCESS_DENIED)
+ return "ACCESS_DENIED";
+ else if (code == ROBOTS_DENIED)
+ return "ROBOTS_DENIED";
+ else if (code == REDIR_EXCEEDED)
+ return "REDIR_EXCEEDED";
+ else if (code == NOTFETCHING)
+ return "NOTFETCHING";
+ else if (code == NOTMODIFIED)
+ return "NOTMODIFIED";
+ else if (code == WOULDBLOCK)
+ return "WOULDBLOCK";
+ else if (code == BLOCKED)
+ return "BLOCKED";
+ return "UNKNOWN_CODE_" + code;
+ }
+
+ public static ProtocolStatus makeStatus(int code) {
+ ProtocolStatus pstatus = new ProtocolStatus();
+ pstatus.setCode(code);
+ pstatus.setLastModified(0);
+ return pstatus;
+ }
+
+ public static ProtocolStatus makeStatus(int code, String message) {
+ ProtocolStatus pstatus = makeStatus(code);
+ pstatus.addToArgs(new Utf8(message));
+ return pstatus;
+ }
+
+ public static ProtocolStatus makeStatus(int code, URL url) {
+ return makeStatus(code, url.toString());
+ }
+
+ public static String getMessage(ProtocolStatus pstatus) {
+ GenericArray<Utf8> args = pstatus.getArgs();
+ if (args == null || args.size() == 0) {
+ return null;
+ }
+ return TableUtil.toString(args.iterator().next());
+ }
+
+ public static String toString(ProtocolStatus status) {
+ if (status == null) {
+ return "(null)";
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append(getName(status.getCode()));
+ sb.append(", args=[");
+ GenericArray<Utf8> args = status.getArgs();
+ if (args != null) {
+ int i = 0;
+ Iterator<Utf8> it = args.iterator();
+ while (it.hasNext()) {
+ if (i > 0) sb.append(',');
+ sb.append(it.next());
+ i++;
+ }
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+}
Propchange:
nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatusUtils.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=987132&r1=987131&r2=987132&view=diff
==============================================================================
---
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
(original)
+++
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
Thu Aug 19 11:31:38 2010
@@ -31,8 +31,8 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.protocol.ProtocolException;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatusCodes;
+import org.apache.nutch.protocol.ProtocolStatusUtils;
import org.apache.nutch.protocol.RobotRules;
-import org.apache.nutch.storage.ProtocolStatusUtils;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.GZIPUtils;
import org.apache.nutch.util.DeflateUtils;
Modified:
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java?rev=987132&r1=987131&r2=987132&view=diff
==============================================================================
---
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
(original)
+++
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
Thu Aug 19 11:31:38 2010
@@ -30,9 +30,9 @@ import org.apache.nutch.protocol.EmptyRo
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatusCodes;
+import org.apache.nutch.protocol.ProtocolStatusUtils;
import org.apache.nutch.protocol.RobotRules;
import org.apache.nutch.storage.ProtocolStatus;
-import org.apache.nutch.storage.ProtocolStatusUtils;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.NutchConfiguration;
Modified:
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java?rev=987132&r1=987131&r2=987132&view=diff
==============================================================================
---
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
(original)
+++
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
Thu Aug 19 11:31:38 2010
@@ -32,9 +32,9 @@ import org.apache.nutch.protocol.EmptyRo
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatusCodes;
+import org.apache.nutch.protocol.ProtocolStatusUtils;
import org.apache.nutch.protocol.RobotRules;
import org.apache.nutch.storage.ProtocolStatus;
-import org.apache.nutch.storage.ProtocolStatusUtils;
import org.apache.nutch.storage.WebPage;
/************************************
