[
https://issues.apache.org/jira/browse/NUTCH-2552?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16446877#comment-16446877
]
ASF GitHub Bot commented on NUTCH-2552:
---------------------------------------
sebastian-nagel closed pull request #315: NUTCH-2552 CrawlDbReader -topN fails
URL: https://github.com/apache/nutch/pull/315
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml
index 405b99f50..8874fc0bc 100644
--- a/conf/nutch-default.xml
+++ b/conf/nutch-default.xml
@@ -985,7 +985,7 @@
<name>fetcher.threads.timeout.divisor</name>
<value>2</value>
<description>(EXPERT)The thread time-out divisor to use. By default threads
have a time-out
- value of mapred.task.timeout / 2. Increase this setting if the fetcher waits
too
+ value of mapreduce.task.timeout / 2. Increase this setting if the fetcher
waits too
long before killing hanged threads. Be careful, a too high setting (+8) will
most likely kill the
fetcher threads prematurely.
</description>
diff --git a/src/java/org/apache/nutch/crawl/CrawlDbReader.java
b/src/java/org/apache/nutch/crawl/CrawlDbReader.java
index dcf5acead..87bf58525 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDbReader.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDbReader.java
@@ -367,7 +367,7 @@ public void reduce(FloatWritable key, Iterable<Text> values,
public void setup(Reducer<FloatWritable, Text, FloatWritable,
Text>.Context context) {
Configuration conf = context.getConfiguration();
- topN = conf.getLong("db.reader.topn", 100) /
Integer.parseInt(conf.get("mapred.job.reduces"));
+ topN = conf.getLong("db.reader.topn", 100) /
Integer.parseInt(conf.get("mapreduce.job.reduces"));
}
public void close() {
diff --git a/src/java/org/apache/nutch/crawl/LinkDb.java
b/src/java/org/apache/nutch/crawl/LinkDb.java
index 37bfb7bfe..e4d689975 100644
--- a/src/java/org/apache/nutch/crawl/LinkDb.java
+++ b/src/java/org/apache/nutch/crawl/LinkDb.java
@@ -307,7 +307,7 @@ private static Job createJob(Configuration config, Path
linkDb,
FileOutputFormat.setOutputPath(job, newLinkDb);
job.setOutputFormatClass(MapFileOutputFormat.class);
- conf.setBoolean("mapred.output.compress", true);
+ conf.setBoolean("mapreduce.output.fileoutputformat.compress", true);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Inlinks.class);
diff --git a/src/java/org/apache/nutch/crawl/LinkDbMerger.java
b/src/java/org/apache/nutch/crawl/LinkDbMerger.java
index f2f0892a6..e773e950e 100644
--- a/src/java/org/apache/nutch/crawl/LinkDbMerger.java
+++ b/src/java/org/apache/nutch/crawl/LinkDbMerger.java
@@ -164,7 +164,7 @@ public static Job createMergeJob(Configuration config, Path
linkDb,
FileOutputFormat.setOutputPath(job, newLinkDb);
job.setOutputFormatClass(MapFileOutputFormat.class);
- conf.setBoolean("mapred.output.compress", true);
+ conf.setBoolean("mapreduce.output.fileoutputformat.compress", true);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Inlinks.class);
diff --git a/src/java/org/apache/nutch/fetcher/Fetcher.java
b/src/java/org/apache/nutch/fetcher/Fetcher.java
index ba34d68a8..5d8654d0d 100644
--- a/src/java/org/apache/nutch/fetcher/Fetcher.java
+++ b/src/java/org/apache/nutch/fetcher/Fetcher.java
@@ -228,7 +228,7 @@ public void run(Context innerContext) throws IOException {
}
// select a timeout that avoids a task timeout
- long timeout = conf.getInt("mapred.task.timeout", 10 * 60 * 1000)
+ long timeout = conf.getInt("mapreduce.task.timeout", 10 * 60 * 1000)
/ timeoutDivisor;
// Used for threshold check, holds pages and bytes processed in the last
diff --git a/src/java/org/apache/nutch/hostdb/ReadHostDb.java
b/src/java/org/apache/nutch/hostdb/ReadHostDb.java
index 408e3eab3..ffbc66114 100644
--- a/src/java/org/apache/nutch/hostdb/ReadHostDb.java
+++ b/src/java/org/apache/nutch/hostdb/ReadHostDb.java
@@ -177,7 +177,7 @@ private void readHostDb(Path hostDb, Path output, boolean
dumpHomepages, boolean
conf.set(HOSTDB_FILTER_EXPRESSION, expr);
}
conf.setBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", false);
- conf.set("mapred.textoutputformat.separator", "\t");
+ conf.set("mapreduce.output.textoutputformat.separator", "\t");
Job job = Job.getInstance(conf);
job.setJobName("ReadHostDb");
diff --git a/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
b/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
index 4e98fcb13..84216478b 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
@@ -330,7 +330,7 @@ public void dumpNodes(Path webGraphDb, DumpType type, long
topN, Path output,
// Set equals-sign as separator for Solr's ExternalFileField
if (asEff) {
- conf.set("mapred.textoutputformat.separator", "=");
+ conf.set("mapreduce.output.textoutputformat.separator", "=");
}
try {
diff --git a/src/java/org/apache/nutch/util/SitemapProcessor.java
b/src/java/org/apache/nutch/util/SitemapProcessor.java
index 70f43727f..ea28550bd 100644
--- a/src/java/org/apache/nutch/util/SitemapProcessor.java
+++ b/src/java/org/apache/nutch/util/SitemapProcessor.java
@@ -433,7 +433,7 @@ public static void usage() {
System.err.println("\t<crawldb>\t\tpath to crawldb where the sitemap urls
would be injected");
System.err.println("\t-hostdb <hostdb>\tpath of a hostdb. Sitemap(s) from
these hosts would be downloaded");
System.err.println("\t-sitemapUrls <url_dir>\tpath to sitemap urls
directory");
- System.err.println("\t-threads <threads>\tNumber of threads created per
mapper to fetch sitemap urls");
+ System.err.println("\t-threads <threads>\tNumber of threads created per
mapper to fetch sitemap urls (default: 8)");
System.err.println("\t-force\t\t\tforce update even if CrawlDb appears to
be locked (CAUTION advised)");
System.err.println("\t-noStrict\t\tBy default Sitemap parser rejects
invalid urls. '-noStrict' disables that.");
System.err.println("\t-noFilter\t\tturn off URLFilters on urls
(optional)");
@@ -452,7 +452,7 @@ public int run(String[] args) throws Exception {
boolean strict = true;
boolean filter = true;
boolean normalize = true;
- int threads = getConf().getInt("mapred.map.multithreadedrunner.threads",
8);
+ int threads = 8;
for (int i = 1; i < args.length; i++) {
if (args[i].equals("-hostdb")) {
diff --git
a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
index 456572b69..e02d904c1 100644
---
a/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
+++
b/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
@@ -200,7 +200,7 @@ private void configureClient() {
// for multi-threaded crawls.
//
--------------------------------------------------------------------------------
params.setMaxTotalConnections(
- conf.getInt("mapred.tasktracker.map.tasks.maximum", 5)
+ conf.getInt("mapreduce.tasktracker.map.tasks.maximum", 5)
* conf.getInt("fetcher.threads.fetch", maxThreadsTotal));
// Also set max connections per host to maxThreadsTotal since all threads
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> CrawlDbReader -topN fails
> -------------------------
>
> Key: NUTCH-2552
> URL: https://issues.apache.org/jira/browse/NUTCH-2552
> Project: Nutch
> Issue Type: Bug
> Components: crawldb
> Affects Versions: 1.15
> Reporter: Sebastian Nagel
> Priority: Critical
> Fix For: 1.15
>
>
> {noformat}
> % bin/nutch readdb crawldb -topN 50 crawldb_topn
> CrawlDb topN: starting (topN=50, min=0.0)
> CrawlDb db: crawl/crawldb
> CrawlDb topN: collecting topN scores.
> CrawlDbReader job did not succeed, job status:FAILED, reason: NA
> Exception in thread "main" java.lang.RuntimeException: CrawlDbReader job did
> not succeed, job status:FAILED, reason: NA
> at
> org.apache.nutch.crawl.CrawlDbReader.processTopNJob(CrawlDbReader.java:853)
> {noformat}
> The hadoop.log shows the reason
> {noformat}
> 2018-04-09 10:04:16,435 WARN mapred.LocalJobRunner - job_local1653923841_0002
> java.lang.Exception: java.lang.NumberFormatException: null
> at
> org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
> at
> org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
> Caused by: java.lang.NumberFormatException: null
> at java.lang.Integer.parseInt(Integer.java:542)
> at java.lang.Integer.parseInt(Integer.java:615)
> at
> org.apache.nutch.crawl.CrawlDbReader$CrawlDbTopNReducer.setup(CrawlDbReader.java:370)
> at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:168)
> at
> org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627)
> {noformat}
> Caused by NUTCH-2375: the property {{mapred.job.reduces}} must be updated by
> {{mapreduce.job.reduces}}.
> Note: Should check all occurrences of this property and similars ones
> ({{mapred.job.*}}).
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
