Author: lewismc
Date: Wed Apr 17 23:20:36 2013
New Revision: 1469099
URL: http://svn.apache.org/r1469099
Log:
NUTCH-1501 Harmonize behavior of parsechecker and indexchecker
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/conf/log4j.properties
nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java
nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java
Modified: nutch/branches/2.x/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Wed Apr 17 23:20:36 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1501 Harmonize behavior of parsechecker and indexchecker (snagel +
lewismc)
+
* NUTCH-1551 Improve WebTableReader field order and display batchId (lewismc)
* NUTCH-1552 possibility of a NPE in index-more plugin (kaveh minooie via
lewismc)
Modified: nutch/branches/2.x/conf/log4j.properties
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/conf/log4j.properties?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
--- nutch/branches/2.x/conf/log4j.properties (original)
+++ nutch/branches/2.x/conf/log4j.properties Wed Apr 17 23:20:36 2013
@@ -38,6 +38,8 @@ log4j.logger.org.apache.nutch.indexer.so
log4j.logger.org.apache.nutch.indexer.DeleteDuplicates=INFO,cmdstdout
log4j.logger.org.apache.nutch.crawl.WebTableReader=INFO,cmdstdout
log4j.logger.org.apache.nutch.host.HostDbReader=INFO,cmdstdout
+log4j.logger.org.apache.nutch.parse.ParserChecker=INFO,cmdstdout
+log4j.logger.org.apache.nutch.indexer.IndexingFiltersChecker=INFO,cmdstdout
log4j.logger.org.apache.nutch=INFO
log4j.logger.org.apache.hadoop=WARN
Modified:
nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
---
nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
(original)
+++
nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
Wed Apr 17 23:20:36 2013
@@ -37,6 +37,7 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.protocol.ProtocolStatusUtils;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,7 +68,7 @@ public class IndexingFiltersChecker exte
return -1;
}
- url = args[0];
+ url = URLUtil.toASCII(args[0]);
if (LOG.isInfoEnabled()) {
LOG.info("fetching: " + url);
Modified: nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java
(original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java Wed
Apr 17 23:20:36 2013
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.SignatureFactory;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolFactory;
@@ -36,10 +37,32 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.Bytes;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.StringUtil;
+import org.apache.nutch.util.URLUtil;
/**
* Parser checker, useful for testing parser.
- *
+ * It also accurately reports possible fetching and
+ * parsing failures and presents protocol status signals to aid
+ * debugging. The tool enables us to retrieve the following data from
+ * any url:
+ * <ol>
+ * <li><tt>contentType</tt>: The URL {@link org.apache.nutch.protocol.Content}
type.</li>
+ * <li><tt>signature</tt>: Digest is used to identify pages (like unique ID)
and is used to remove
+ * duplicates during the dedup procedure.
+ * It is calculated using {@link org.apache.nutch.crawl.MD5Signature} or
+ * {@link org.apache.nutch.crawl.TextProfileSignature}.</li>
+ * <li><tt>Version</tt>: From {@link org.apache.nutch.parse.ParseData}.</li>
+ * <li><tt>Status</tt>: From {@link org.apache.nutch.parse.ParseData}.</li>
+ * <li><tt>Title</tt>: of the URL</li>
+ * <li><tt>Outlinks</tt>: associated with the URL</li>
+ * <li><tt>Content Metadata</tt>: such as <i>X-AspNet-Version</i>, <i>Date</i>,
+ * <i>Content-length</i>, <i>servedBy</i>, <i>Content-Type</i>,
<i>Cache-Control</>, etc.</li>
+ * <li><tt>Parse Metadata</tt>: such as <i>CharEncodingForConversion</i>,
+ * <i>OriginalCharEncoding</i>, <i>language</i>, etc.</li>
+ * <li><tt>ParseText</tt>: The page parse text which varies in length
depdnecing on
+ * <code>content.length</code> configuration.</li>
+ * </ol>
* @author John Xing
*/
@@ -60,7 +83,7 @@ public class ParserChecker implements To
String usage = "Usage: ParserChecker [-dumpText] [-forceAs mimeType] url";
if (args.length == 0) {
- System.err.println(usage);
+ LOG.error(usage);
return (-1);
}
@@ -71,10 +94,10 @@ public class ParserChecker implements To
} else if (args[i].equals("-dumpText")) {
dumpText = true;
} else if (i != args.length - 1) {
- System.err.println(usage);
+ LOG.error(usage);
System.exit(-1);
} else {
- url = args[i];
+ url = URLUtil.toASCII(args[i]);
}
}
@@ -110,15 +133,10 @@ public class ParserChecker implements To
}
if (contentType == null) {
- System.err.println("");
+ LOG.error("Failed to determine content type!");
return (-1);
}
- if (LOG.isInfoEnabled()) {
- LOG.info("parsing: " + url);
- LOG.info("contentType: " + contentType);
- }
-
page.setContentType(new Utf8(contentType));
if (ParserJob.isTruncated(url, page)) {
@@ -128,13 +146,23 @@ public class ParserChecker implements To
Parse parse = new ParseUtil(conf).parse(url, page);
if (parse == null) {
- System.err.println("Problem with parse - check log");
+ LOG.error("Problem with parse - check log");
return (-1);
}
+
+ // Calculate the signature
+ byte[] signature =
SignatureFactory.getSignature(getConf()).calculate(page);
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("parsing: " + url);
+ LOG.info("contentType: " + contentType);
+ LOG.info("signature: " + StringUtil.toHexString(signature));
+ }
+
- System.out.print("---------\nUrl\n---------------\n");
+ LOG.info("---------\nUrl\n---------------\n");
System.out.print(url + "\n");
- System.out.print("---------\nMetadata\n---------\n");
+ LOG.info("---------\nMetadata\n---------\n");
Map<Utf8, ByteBuffer> metadata = page.getMetadata();
StringBuffer sb = new StringBuffer();
if (metadata != null) {
@@ -148,7 +176,7 @@ public class ParserChecker implements To
System.out.print(sb.toString());
}
if (dumpText) {
- System.out.print("---------\nParseText\n---------\n");
+ LOG.info("---------\nParseText\n---------\n");
System.out.print(parse.getText());
}
@@ -170,4 +198,5 @@ public class ParserChecker implements To
args);
System.exit(res);
}
+
}
Modified: nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java
URL:
http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java Wed Apr 17
23:20:36 2013
@@ -18,7 +18,7 @@
package org.apache.nutch.util;
import java.net.MalformedURLException;
-import java.net.URL;
+import java.net.*;
import java.util.regex.Pattern;
import org.apache.nutch.util.domain.DomainSuffix;
@@ -333,6 +333,43 @@ public class URLUtil {
}
}
+ public static String toASCII(String url) {
+ try {
+ URL u = new URL(url);
+ URI p = new URI(u.getProtocol(),
+ null,
+ IDN.toASCII(u.getHost()),
+ u.getPort(),
+ u.getPath(),
+ u.getQuery(),
+ u.getRef());
+
+ return p.toString();
+ }
+ catch (Exception e) {
+ return null;
+ }
+ }
+
+ public static String toUNICODE(String url) {
+ try {
+ URL u = new URL(url);
+ URI p = new URI(u.getProtocol(),
+ null,
+ IDN.toUnicode(u.getHost()),
+ u.getPort(),
+ u.getPath(),
+ u.getQuery(),
+ u.getRef());
+
+ return p.toString();
+ }
+ catch (Exception e) {
+ return null;
+ }
+ }
+
+
/** For testing */
public static void main(String[] args){
