[
https://issues.apache.org/jira/browse/NUTCH-2435?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16211766#comment-16211766
]
ASF GitHub Bot commented on NUTCH-2435:
---------------------------------------
sebastian-nagel closed pull request #225: NUTCH-2435 - New parameter
"parser.store.text"
URL: https://github.com/apache/nutch/pull/225
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml
index c406907c5..587386140 100644
--- a/conf/nutch-default.xml
+++ b/conf/nutch-default.xml
@@ -1372,6 +1372,13 @@ CAUTION: Set the parser.timeout to -1 or a bigger value
than 30, when using this
</description>
</property>
+<property>
+ <name>parser.store.text</name>
+ <value>true</value>
+ <description>If true (default value), parser will store parse text
(parse_text directory within the segment).</description>
+</property>
+
+
<!--
<property>
<name>tika.htmlmapper.classname</name>
diff --git a/src/java/org/apache/nutch/parse/ParseOutputFormat.java
b/src/java/org/apache/nutch/parse/ParseOutputFormat.java
index 6e84b1265..2c8396a75 100644
--- a/src/java/org/apache/nutch/parse/ParseOutputFormat.java
+++ b/src/java/org/apache/nutch/parse/ParseOutputFormat.java
@@ -111,7 +111,10 @@ public void checkOutputSpecs(FileSystem fs, JobConf job)
throws IOException {
"db.ignore.external.links", false);
final String ignoreExternalLinksMode = job.get(
"db.ignore.external.links.mode", "byHost");
-
+ // NUTCH-2435 - parameter "parser.store.text" allowing to choose whether to
+ // store 'parse_text' directory or not:
+ final boolean storeText = job.getBoolean("parser.store.text", true);
+
int maxOutlinksPerPage = job.getInt("db.max.outlinks.per.page", 100);
final boolean isParsing = job.getBoolean("fetcher.parse", true);
final int maxOutlinks = (maxOutlinksPerPage < 0) ? Integer.MAX_VALUE
@@ -128,14 +131,22 @@ public void checkOutputSpecs(FileSystem fs, JobConf job)
throws IOException {
.split(" *, *");
// textOut Options
- Option tKeyClassOpt = (Option) MapFile.Writer.keyClass(Text.class);
- org.apache.hadoop.io.SequenceFile.Writer.Option tValClassOpt =
SequenceFile.Writer.valueClass(ParseText.class);
- org.apache.hadoop.io.SequenceFile.Writer.Option tProgressOpt =
SequenceFile.Writer.progressable(progress);
- org.apache.hadoop.io.SequenceFile.Writer.Option tCompOpt =
SequenceFile.Writer.compression(CompressionType.RECORD);
-
- final MapFile.Writer textOut = new MapFile.Writer(job, text,
- tKeyClassOpt, tValClassOpt, tCompOpt, tProgressOpt);
-
+ final MapFile.Writer textOut;
+ if (storeText) {
+ Option tKeyClassOpt = (Option) MapFile.Writer.keyClass(Text.class);
+ org.apache.hadoop.io.SequenceFile.Writer.Option tValClassOpt =
SequenceFile.Writer
+ .valueClass(ParseText.class);
+ org.apache.hadoop.io.SequenceFile.Writer.Option tProgressOpt =
SequenceFile.Writer
+ .progressable(progress);
+ org.apache.hadoop.io.SequenceFile.Writer.Option tCompOpt =
SequenceFile.Writer
+ .compression(CompressionType.RECORD);
+
+ textOut = new MapFile.Writer(job, text, tKeyClassOpt, tValClassOpt,
+ tCompOpt, tProgressOpt);
+ } else {
+ textOut = null;
+ }
+
// dataOut Options
Option dKeyClassOpt = (Option) MapFile.Writer.keyClass(Text.class);
org.apache.hadoop.io.SequenceFile.Writer.Option dValClassOpt =
SequenceFile.Writer.valueClass(ParseData.class);
@@ -162,7 +173,9 @@ public void write(Text key, Parse parse) throws IOException
{
String fromUrl = key.toString();
// host or domain name of the source URL
String origin = null;
- textOut.append(key, new ParseText(parse.getText()));
+ if (textOut != null) {
+ textOut.append(key, new ParseText(parse.getText()));
+ }
ParseData parseData = parse.getData();
// recover the signature prepared by Fetcher or ParseSegment
@@ -311,7 +324,8 @@ public void write(Text key, Parse parse) throws IOException
{
}
public void close(Reporter reporter) throws IOException {
- textOut.close();
+ if (textOut != null)
+ textOut.close();
dataOut.close();
crawlOut.close();
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> New configuration allowing to choose whether to store 'parse_text' directory
> or not.
> ------------------------------------------------------------------------------------
>
> Key: NUTCH-2435
> URL: https://issues.apache.org/jira/browse/NUTCH-2435
> Project: Nutch
> Issue Type: New Feature
> Components: parser
> Affects Versions: 1.13
> Environment: Apach Nutch 1.13
> Reporter: Marcos Bori
>
> Whenever a page is parsed, one of the outputs is the directory 'parse_text'.
> It is intended to be used at the indexing phase so the page can be searched
> from a search engine such as Solr.
> In my special crawling case, I don't need to index the page contents.
> Therefore, creating and filing the 'parse_text' is not required for me. To
> optimize performance, I don't want the crawler to store this information to
> the filesystem.
> I propose a new parameter "parser.store.text" allowing to choose whether to
> store 'parse_text' directory or not. Its default value, of course, is "true".
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
