This is an automated email from the ASF dual-hosted git repository.
snagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 22e668d NUTCH-2774 Annotate methods implementing the Hadoop API by
@Override - annotate classes implementing Hadoop interfaces - annotate few
classes implementing Nutch interfaces - remove empty method implementations
when super classes already provide a default implementation
new 3cffe3b Merge pull request #502 from
sebastian-nagel/NUTCH-2774-override-annotations
22e668d is described below
commit 22e668dcbbbc41e1a9d0c0304cc08e87be581e9c
Author: Sebastian Nagel <[email protected]>
AuthorDate: Fri Feb 28 13:42:45 2020 +0100
NUTCH-2774 Annotate methods implementing the Hadoop API by @Override
- annotate classes implementing Hadoop interfaces
- annotate few classes implementing Nutch interfaces
- remove empty method implementations when super classes already
provide a default implementation
---
src/java/org/apache/nutch/crawl/CrawlDb.java | 1 +
src/java/org/apache/nutch/crawl/CrawlDbFilter.java | 6 ++----
src/java/org/apache/nutch/crawl/CrawlDbMerger.java | 7 +++----
src/java/org/apache/nutch/crawl/CrawlDbReader.java | 10 ++++++++++
src/java/org/apache/nutch/crawl/CrawlDbReducer.java | 5 ++---
src/java/org/apache/nutch/crawl/DeduplicationJob.java | 1 +
src/java/org/apache/nutch/crawl/Generator.java | 5 +++++
src/java/org/apache/nutch/crawl/Injector.java | 6 ++++++
src/java/org/apache/nutch/crawl/LinkDb.java | 5 ++++-
src/java/org/apache/nutch/crawl/LinkDbFilter.java | 5 ++---
src/java/org/apache/nutch/crawl/LinkDbMerger.java | 6 +++---
src/java/org/apache/nutch/crawl/LinkDbReader.java | 2 ++
src/java/org/apache/nutch/crawl/URLPartitioner.java | 3 ---
src/java/org/apache/nutch/fetcher/Fetcher.java | 1 +
src/java/org/apache/nutch/hostdb/ReadHostDb.java | 3 +++
src/java/org/apache/nutch/hostdb/UpdateHostDb.java | 1 +
src/java/org/apache/nutch/indexer/CleaningJob.java | 1 +
src/java/org/apache/nutch/indexer/IndexerMapReduce.java | 4 +---
.../org/apache/nutch/indexer/IndexingFiltersChecker.java | 1 +
src/java/org/apache/nutch/indexer/IndexingJob.java | 1 +
src/java/org/apache/nutch/net/URLFilterChecker.java | 1 +
src/java/org/apache/nutch/net/URLNormalizerChecker.java | 1 +
src/java/org/apache/nutch/parse/ParseOutputFormat.java | 7 ++++---
src/java/org/apache/nutch/parse/ParseSegment.java | 6 ++----
src/java/org/apache/nutch/parse/ParserChecker.java | 1 +
.../org/apache/nutch/scoring/webgraph/LinkDumper.java | 1 +
src/java/org/apache/nutch/scoring/webgraph/LinkRank.java | 12 +++---------
.../org/apache/nutch/scoring/webgraph/NodeDumper.java | 1 +
.../org/apache/nutch/scoring/webgraph/ScoreUpdater.java | 1 +
src/java/org/apache/nutch/scoring/webgraph/WebGraph.java | 15 ++++++---------
src/java/org/apache/nutch/segment/SegmentMerger.java | 4 +---
src/java/org/apache/nutch/tools/Benchmark.java | 1 +
src/java/org/apache/nutch/tools/FreeGenerator.java | 1 +
src/java/org/apache/nutch/tools/warc/WARCExporter.java | 12 +++---------
src/java/org/apache/nutch/util/CrawlCompletionStats.java | 5 +++++
.../org/apache/nutch/util/ProtocolStatusStatistics.java | 5 ++++-
src/java/org/apache/nutch/util/SitemapProcessor.java | 5 +++++
.../org/apache/nutch/util/domain/DomainStatistics.java | 5 +++++
.../apache/nutch/indexwriter/dummy/DummyIndexWriter.java | 2 ++
.../apache/nutch/indexwriter/solr/SolrIndexWriter.java | 3 +++
40 files changed, 101 insertions(+), 62 deletions(-)
diff --git a/src/java/org/apache/nutch/crawl/CrawlDb.java
b/src/java/org/apache/nutch/crawl/CrawlDb.java
index 8cd5e3e..da654ed 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDb.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDb.java
@@ -229,6 +229,7 @@ public class CrawlDb extends NutchTool implements Tool {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 1) {
System.err
diff --git a/src/java/org/apache/nutch/crawl/CrawlDbFilter.java
b/src/java/org/apache/nutch/crawl/CrawlDbFilter.java
index 5da9951..8abe4b4 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDbFilter.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDbFilter.java
@@ -52,7 +52,7 @@ public class CrawlDbFilter extends
private static final Logger LOG = LoggerFactory
.getLogger(MethodHandles.lookup().lookupClass());
-
+ @Override
public void setup(Mapper<Text, CrawlDatum, Text, CrawlDatum>.Context
context) {
Configuration conf = context.getConfiguration();
urlFiltering = conf.getBoolean(URL_FILTERING, false);
@@ -69,11 +69,9 @@ public class CrawlDbFilter extends
}
}
- public void close() {
- }
-
private Text newKey = new Text();
+ @Override
public void map(Text key, CrawlDatum value,
Context context) throws IOException, InterruptedException {
diff --git a/src/java/org/apache/nutch/crawl/CrawlDbMerger.java
b/src/java/org/apache/nutch/crawl/CrawlDbMerger.java
index 7c6ef93..bc77091 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDbMerger.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDbMerger.java
@@ -71,15 +71,14 @@ public class CrawlDbMerger extends Configured implements
Tool {
Reducer<Text, CrawlDatum, Text, CrawlDatum> {
private FetchSchedule schedule;
- public void close() throws IOException {
- }
-
+ @Override
public void setup(
Reducer<Text, CrawlDatum, Text, CrawlDatum>.Context context) {
Configuration conf = context.getConfiguration();
schedule = FetchScheduleFactory.getFetchSchedule(conf);
}
+ @Override
public void reduce(Text key, Iterable<CrawlDatum> values,
Context context)
throws IOException, InterruptedException {
@@ -120,7 +119,6 @@ public class CrawlDbMerger extends Configured implements
Tool {
}
public CrawlDbMerger() {
-
}
public CrawlDbMerger(Configuration conf) {
@@ -200,6 +198,7 @@ public class CrawlDbMerger extends Configured implements
Tool {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err
diff --git a/src/java/org/apache/nutch/crawl/CrawlDbReader.java
b/src/java/org/apache/nutch/crawl/CrawlDbReader.java
index b9200e7..a0831da 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDbReader.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDbReader.java
@@ -167,6 +167,7 @@ public class CrawlDbReader extends AbstractChecker
implements Closeable {
}
}
+ @Override
public synchronized void write(Text key, CrawlDatum value)
throws IOException {
out.writeByte('"');
@@ -212,12 +213,14 @@ public class CrawlDbReader extends AbstractChecker
implements Closeable {
out.writeByte('\n');
}
+ @Override
public synchronized void close(TaskAttemptContext context)
throws IOException {
out.close();
}
}
+ @Override
public RecordWriter<Text, CrawlDatum> getRecordWriter(
TaskAttemptContext context) throws IOException {
String name = getUniqueFile(context, "part", "");
@@ -243,6 +246,7 @@ public class CrawlDbReader extends AbstractChecker
implements Closeable {
jsonWriter = jsonMapper.writer(new JsonIndenter());
}
+ @Override
public synchronized void write(Text key, CrawlDatum value)
throws IOException {
Map<String, Object> data = new LinkedHashMap<String, Object>();
@@ -275,12 +279,14 @@ public class CrawlDbReader extends AbstractChecker
implements Closeable {
out.writeByte('\n');
}
+ @Override
public synchronized void close(TaskAttemptContext context)
throws IOException {
out.close();
}
}
+ @Override
public RecordWriter<Text, CrawlDatum> getRecordWriter(
TaskAttemptContext context) throws IOException {
String name = getUniqueFile(context, "part", "");
@@ -343,6 +349,8 @@ public class CrawlDbReader extends AbstractChecker
implements Closeable {
public static class CrawlDbStatReducer
extends Reducer<Text, NutchWritable, Text, NutchWritable> {
+
+ @Override
public void setup(
Reducer<Text, NutchWritable, Text, NutchWritable>.Context context) {
}
@@ -474,6 +482,7 @@ public class CrawlDbReader extends AbstractChecker
implements Closeable {
}
}
+ @Override
public void close() {
closeReaders();
}
@@ -976,6 +985,7 @@ public class CrawlDbReader extends AbstractChecker
implements Closeable {
}
+ @Override
public int run(String[] args) throws IOException, InterruptedException,
ClassNotFoundException, Exception {
@SuppressWarnings("resource")
diff --git a/src/java/org/apache/nutch/crawl/CrawlDbReducer.java
b/src/java/org/apache/nutch/crawl/CrawlDbReducer.java
index feba08a..984a02d 100644
--- a/src/java/org/apache/nutch/crawl/CrawlDbReducer.java
+++ b/src/java/org/apache/nutch/crawl/CrawlDbReducer.java
@@ -49,6 +49,7 @@ public class CrawlDbReducer extends
private int maxInterval;
private FetchSchedule schedule;
+ @Override
public void setup(Reducer<Text, CrawlDatum, Text, CrawlDatum>.Context
context) {
Configuration conf = context.getConfiguration();
retryMax = conf.getInt("db.fetch.retry.max", 3);
@@ -60,9 +61,7 @@ public class CrawlDbReducer extends
linked = new InlinkPriorityQueue(maxLinks);
}
- public void close() {
- }
-
+ @Override
public void reduce(Text key, Iterable<CrawlDatum> values,
Context context) throws IOException, InterruptedException {
diff --git a/src/java/org/apache/nutch/crawl/DeduplicationJob.java
b/src/java/org/apache/nutch/crawl/DeduplicationJob.java
index 9b01411..04b19e3 100644
--- a/src/java/org/apache/nutch/crawl/DeduplicationJob.java
+++ b/src/java/org/apache/nutch/crawl/DeduplicationJob.java
@@ -266,6 +266,7 @@ public class DeduplicationJob extends NutchTool implements
Tool {
}
}
+ @Override
public int run(String[] args) throws IOException {
if (args.length < 1) {
System.err.println("Usage: DeduplicationJob <crawldb> [-group
<none|host|domain>] [-compareOrder
<score>,<fetchTime>,<httpsOverHttp>,<urlLength>]");
diff --git a/src/java/org/apache/nutch/crawl/Generator.java
b/src/java/org/apache/nutch/crawl/Generator.java
index 1c5e4d5..be64695 100644
--- a/src/java/org/apache/nutch/crawl/Generator.java
+++ b/src/java/org/apache/nutch/crawl/Generator.java
@@ -578,6 +578,7 @@ public class Generator extends NutchTool implements Tool {
public static class SelectorInverseMapper
extends Mapper<FloatWritable, SelectorEntry, Text, SelectorEntry> {
+ @Override
public void map(FloatWritable key, SelectorEntry value, Context context)
throws IOException, InterruptedException {
SelectorEntry entry = value;
@@ -588,6 +589,7 @@ public class Generator extends NutchTool implements Tool {
public static class PartitionReducer
extends Reducer<Text, SelectorEntry, Text, CrawlDatum> {
+ @Override
public void reduce(Text key, Iterable<SelectorEntry> values,
Context context) throws IOException, InterruptedException {
// if using HashComparator, we get only one input key in case of
@@ -606,6 +608,7 @@ public class Generator extends NutchTool implements Tool {
}
@SuppressWarnings("rawtypes")
+ @Override
public int compare(WritableComparable a, WritableComparable b) {
Text url1 = (Text) a;
Text url2 = (Text) b;
@@ -614,6 +617,7 @@ public class Generator extends NutchTool implements Tool {
return (hash1 < hash2 ? -1 : (hash1 == hash2 ? 0 : 1));
}
+ @Override
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
int hash1 = hash(b1, s1, l1);
int hash2 = hash(b2, s2, l2);
@@ -989,6 +993,7 @@ public class Generator extends NutchTool implements Tool {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.out.println(
diff --git a/src/java/org/apache/nutch/crawl/Injector.java
b/src/java/org/apache/nutch/crawl/Injector.java
index 7d4ee84..9f898e6 100644
--- a/src/java/org/apache/nutch/crawl/Injector.java
+++ b/src/java/org/apache/nutch/crawl/Injector.java
@@ -128,6 +128,7 @@ public class Injector extends NutchTool implements Tool {
private String scope;
private boolean filterNormalizeAll = false;
+ @Override
public void setup(Context context) {
Configuration conf = context.getConfiguration();
boolean normalize = conf.getBoolean(CrawlDbFilter.URL_NORMALIZING, true);
@@ -205,6 +206,7 @@ public class Injector extends NutchTool implements Tool {
}
}
+ @Override
public void map(Text key, Writable value, Context context)
throws IOException, InterruptedException {
if (value instanceof Text) {
@@ -277,6 +279,7 @@ public class Injector extends NutchTool implements Tool {
private CrawlDatum old = new CrawlDatum();
private CrawlDatum injected = new CrawlDatum();
+ @Override
public void setup(Context context) {
Configuration conf = context.getConfiguration();
interval = conf.getInt("db.fetch.interval.default", 2592000);
@@ -302,6 +305,7 @@ public class Injector extends NutchTool implements Tool {
*
* For more details @see NUTCH-1405
*/
+ @Override
public void reduce(Text key, Iterable<CrawlDatum> values, Context context)
throws IOException, InterruptedException {
@@ -535,6 +539,7 @@ public class Injector extends NutchTool implements Tool {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
usage();
@@ -578,6 +583,7 @@ public class Injector extends NutchTool implements Tool {
/**
* Used by the Nutch REST service
*/
+ @Override
public Map<String, Object> run(Map<String, Object> args, String crawlId)
throws Exception {
if(args.size()<1){
diff --git a/src/java/org/apache/nutch/crawl/LinkDb.java
b/src/java/org/apache/nutch/crawl/LinkDb.java
index b32e64f..51e880d 100644
--- a/src/java/org/apache/nutch/crawl/LinkDb.java
+++ b/src/java/org/apache/nutch/crawl/LinkDb.java
@@ -84,6 +84,7 @@ public class LinkDb extends NutchTool implements Tool {
private URLFilters urlFilters;
private URLNormalizers urlNormalizers;
+ @Override
public void setup(Mapper<Text, ParseData, Text, Inlinks>.Context context) {
Configuration conf = context.getConfiguration();
maxAnchorLength = conf.getInt("linkdb.max.anchor.length", 100);
@@ -98,7 +99,8 @@ public class LinkDb extends NutchTool implements Tool {
}
}
- public void map(Text key, ParseData parseData,
+ @Override
+ public void map(Text key, ParseData parseData,
Context context)
throws IOException, InterruptedException {
String fromUrl = key.toString();
@@ -333,6 +335,7 @@ public class LinkDb extends NutchTool implements Tool {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err
diff --git a/src/java/org/apache/nutch/crawl/LinkDbFilter.java
b/src/java/org/apache/nutch/crawl/LinkDbFilter.java
index 33895f9..ed9151a 100644
--- a/src/java/org/apache/nutch/crawl/LinkDbFilter.java
+++ b/src/java/org/apache/nutch/crawl/LinkDbFilter.java
@@ -56,6 +56,7 @@ public class LinkDbFilter extends Mapper<Text, Inlinks, Text,
Inlinks> {
private Text newKey = new Text();
+ @Override
public void setup(Mapper<Text, Inlinks, Text, Inlinks>.Context context) {
Configuration conf = context.getConfiguration();
filter = conf.getBoolean(URL_FILTERING, false);
@@ -69,9 +70,7 @@ public class LinkDbFilter extends Mapper<Text, Inlinks, Text,
Inlinks> {
}
}
- public void close() {
- }
-
+ @Override
public void map(Text key, Inlinks value, Context context)
throws IOException, InterruptedException {
String url = key.toString();
diff --git a/src/java/org/apache/nutch/crawl/LinkDbMerger.java
b/src/java/org/apache/nutch/crawl/LinkDbMerger.java
index d5942be..059dbcd 100644
--- a/src/java/org/apache/nutch/crawl/LinkDbMerger.java
+++ b/src/java/org/apache/nutch/crawl/LinkDbMerger.java
@@ -82,11 +82,13 @@ public class LinkDbMerger extends Configured implements
Tool {
private int maxInlinks;
+ @Override
public void setup(Reducer<Text, Inlinks, Text, Inlinks>.Context context) {
Configuration conf = context.getConfiguration();
maxInlinks = conf.getInt("linkdb.max.inlinks", 10000);
}
+ @Override
public void reduce(Text key, Iterable<Inlinks> values, Context context)
throws IOException, InterruptedException {
@@ -108,9 +110,6 @@ public class LinkDbMerger extends Configured implements
Tool {
}
}
- public void close() throws IOException {
- }
-
public void merge(Path output, Path[] dbs, boolean normalize, boolean filter)
throws Exception {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
@@ -183,6 +182,7 @@ public class LinkDbMerger extends Configured implements
Tool {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err
diff --git a/src/java/org/apache/nutch/crawl/LinkDbReader.java
b/src/java/org/apache/nutch/crawl/LinkDbReader.java
index 6ea3c26..8b649c8 100644
--- a/src/java/org/apache/nutch/crawl/LinkDbReader.java
+++ b/src/java/org/apache/nutch/crawl/LinkDbReader.java
@@ -115,6 +115,7 @@ public class LinkDbReader extends AbstractChecker
implements Closeable {
new Inlinks());
}
+ @Override
public void close() throws IOException {
if (readers != null) {
for (int i = 0; i < readers.length; i++) {
@@ -220,6 +221,7 @@ public class LinkDbReader extends AbstractChecker
implements Closeable {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err
diff --git a/src/java/org/apache/nutch/crawl/URLPartitioner.java
b/src/java/org/apache/nutch/crawl/URLPartitioner.java
index 80b4fab..d9e6c4c 100644
--- a/src/java/org/apache/nutch/crawl/URLPartitioner.java
+++ b/src/java/org/apache/nutch/crawl/URLPartitioner.java
@@ -71,9 +71,6 @@ public class URLPartitioner extends Partitioner<Text,
Writable> implements Confi
return conf;
}
- public void close() {
- }
-
/** Hash by host or domain name or IP address. */
public int getPartition(Text key, Writable value, int numReduceTasks) {
String urlString = key.toString();
diff --git a/src/java/org/apache/nutch/fetcher/Fetcher.java
b/src/java/org/apache/nutch/fetcher/Fetcher.java
index e5250ae..e489ae8 100644
--- a/src/java/org/apache/nutch/fetcher/Fetcher.java
+++ b/src/java/org/apache/nutch/fetcher/Fetcher.java
@@ -521,6 +521,7 @@ public class Fetcher extends NutchTool implements Tool {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
String usage = "Usage: Fetcher <segment> [-threads n]";
diff --git a/src/java/org/apache/nutch/hostdb/ReadHostDb.java
b/src/java/org/apache/nutch/hostdb/ReadHostDb.java
index daf013f..62bf3a7 100644
--- a/src/java/org/apache/nutch/hostdb/ReadHostDb.java
+++ b/src/java/org/apache/nutch/hostdb/ReadHostDb.java
@@ -69,6 +69,7 @@ public class ReadHostDb extends Configured implements Tool {
protected Text emptyText = new Text();
protected Expression expr = null;
+ @Override
public void setup(Context context) {
dumpHomepages =
context.getConfiguration().getBoolean(HOSTDB_DUMP_HOMEPAGES, false);
dumpHostnames =
context.getConfiguration().getBoolean(HOSTDB_DUMP_HOSTNAMES, false);
@@ -87,6 +88,7 @@ public class ReadHostDb extends Configured implements Tool {
}
}
+ @Override
public void map(Text key, HostDatum datum, Context context) throws
IOException, InterruptedException {
if (fieldHeader && !dumpHomepages && !dumpHostnames) {
context.write(new Text("hostname"), new
Text("unfetched\tfetched\tgone\tredirTemp\tredirPerm\tnotModified\tnumRecords\tdnsFail\tcnxFail\tsumFail\tscore\tlastCheck\thomepage\tmetadata"));
@@ -242,6 +244,7 @@ public class ReadHostDb extends Configured implements Tool {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err.println("Usage: ReadHostDb <hostdb> [-get <url>] [<output>
[-dumpHomepages | -dumpHostnames | -expr <expr.>]]");
diff --git a/src/java/org/apache/nutch/hostdb/UpdateHostDb.java
b/src/java/org/apache/nutch/hostdb/UpdateHostDb.java
index 7066f7e..60c9fa6 100644
--- a/src/java/org/apache/nutch/hostdb/UpdateHostDb.java
+++ b/src/java/org/apache/nutch/hostdb/UpdateHostDb.java
@@ -160,6 +160,7 @@ public class UpdateHostDb extends Configured implements
Tool {
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err.println("Usage: UpdateHostDb -hostdb <hostdb> " +
diff --git a/src/java/org/apache/nutch/indexer/CleaningJob.java
b/src/java/org/apache/nutch/indexer/CleaningJob.java
index 9b49653..ca1198e 100644
--- a/src/java/org/apache/nutch/indexer/CleaningJob.java
+++ b/src/java/org/apache/nutch/indexer/CleaningJob.java
@@ -180,6 +180,7 @@ public class CleaningJob implements Tool {
+ TimingUtil.elapsedTime(start, end));
}
+ @Override
public int run(String[] args) throws IOException {
if (args.length < 1) {
String usage = "Usage: CleaningJob <crawldb> [-noCommit]";
diff --git a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
index cfb6dea..856b99a 100644
--- a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
+++ b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
@@ -238,6 +238,7 @@ public class IndexerMapReduce extends Configured {
}
}
+ @Override
public void reduce(Text key, Iterable<NutchWritable> values,
Context context) throws IOException, InterruptedException {
Inlinks inlinks = null;
@@ -430,9 +431,6 @@ public class IndexerMapReduce extends Configured {
}
}
- public void close() throws IOException {
- }
-
public static void initMRJob(Path crawlDb, Path linkDb,
Collection<Path> segments, Job job, boolean addBinaryContent) throws
IOException{
diff --git a/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
b/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
index a43ccb1..4f849a0 100644
--- a/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
+++ b/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
@@ -65,6 +65,7 @@ public class IndexingFiltersChecker extends AbstractChecker {
private static final Logger LOG = LoggerFactory
.getLogger(MethodHandles.lookup().lookupClass());
+ @Override
public int run(String[] args) throws Exception {
String url = null;
diff --git a/src/java/org/apache/nutch/indexer/IndexingJob.java
b/src/java/org/apache/nutch/indexer/IndexingJob.java
index e476adc..0966276 100644
--- a/src/java/org/apache/nutch/indexer/IndexingJob.java
+++ b/src/java/org/apache/nutch/indexer/IndexingJob.java
@@ -211,6 +211,7 @@ public class IndexingJob extends NutchTool implements Tool {
System.err.println("");
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length == 0) {
usage();
diff --git a/src/java/org/apache/nutch/net/URLFilterChecker.java
b/src/java/org/apache/nutch/net/URLFilterChecker.java
index c1d1093..4e613d0 100644
--- a/src/java/org/apache/nutch/net/URLFilterChecker.java
+++ b/src/java/org/apache/nutch/net/URLFilterChecker.java
@@ -30,6 +30,7 @@ public class URLFilterChecker extends AbstractChecker {
private URLFilters filters = null;
+ @Override
public int run(String[] args) throws Exception {
usage = "Usage: URLFilterChecker [-Dproperty=value]... [-filterName
filterName] (-stdin | -listen <port> [-keepClientCnxOpen]) \n"
+ "\n -filterName\tURL filter plugin name (eg. urlfilter-regex) to
check,"
diff --git a/src/java/org/apache/nutch/net/URLNormalizerChecker.java
b/src/java/org/apache/nutch/net/URLNormalizerChecker.java
index ee25f2f..fa0baa2 100644
--- a/src/java/org/apache/nutch/net/URLNormalizerChecker.java
+++ b/src/java/org/apache/nutch/net/URLNormalizerChecker.java
@@ -31,6 +31,7 @@ public class URLNormalizerChecker extends AbstractChecker {
private String scope = URLNormalizers.SCOPE_DEFAULT;
URLNormalizers normalizers;
+ @Override
public int run(String[] args) throws Exception {
usage = "Usage: URLNormalizerChecker [-Dproperty=value]... [-normalizer
<normalizerName>] [-scope <scope>] (-stdin | -listen <port>
[-keepClientCnxOpen])\n"
+ "\n -normalizer\tURL normalizer plugin (eg. urlnormalizer-basic) to
check,"
diff --git a/src/java/org/apache/nutch/parse/ParseOutputFormat.java
b/src/java/org/apache/nutch/parse/ParseOutputFormat.java
index 4bc0853..fcaa1d1 100644
--- a/src/java/org/apache/nutch/parse/ParseOutputFormat.java
+++ b/src/java/org/apache/nutch/parse/ParseOutputFormat.java
@@ -97,6 +97,7 @@ public class ParseOutputFormat extends OutputFormat<Text,
Parse> {
}
}
+ @Override
public OutputCommitter getOutputCommitter(TaskAttemptContext context)
throws IOException {
Path path = FileOutputFormat.getOutputPath(context);
@@ -108,9 +109,6 @@ public class ParseOutputFormat extends OutputFormat<Text,
Parse> {
Configuration conf = context.getConfiguration();
Path out = FileOutputFormat.getOutputPath(context);
FileSystem fs = out.getFileSystem(context.getConfiguration());
- if ((out == null) && (context.getNumReduceTasks() != 0)) {
- throw new IOException("Output directory not set in JobContext.");
- }
if (fs == null) {
fs = out.getFileSystem(conf);
}
@@ -132,6 +130,7 @@ public class ParseOutputFormat extends OutputFormat<Text,
Parse> {
return result.toString();
}
+ @Override
public RecordWriter<Text, Parse> getRecordWriter(TaskAttemptContext context)
throws IOException {
Configuration conf = context.getConfiguration();
@@ -216,6 +215,7 @@ public class ParseOutputFormat extends OutputFormat<Text,
Parse> {
return new RecordWriter<Text, Parse>() {
+ @Override
public void write(Text key, Parse parse) throws IOException {
String fromUrl = key.toString();
@@ -374,6 +374,7 @@ public class ParseOutputFormat extends OutputFormat<Text,
Parse> {
}
}
+ @Override
public void close(TaskAttemptContext context) throws IOException {
if (textOut != null)
textOut.close();
diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java
b/src/java/org/apache/nutch/parse/ParseSegment.java
index 9a92ced..9bd1f03 100644
--- a/src/java/org/apache/nutch/parse/ParseSegment.java
+++ b/src/java/org/apache/nutch/parse/ParseSegment.java
@@ -90,10 +90,6 @@ public class ParseSegment extends NutchTool implements Tool {
}
@Override
- public void cleanup(Context context){
- }
-
- @Override
public void map(WritableComparable<?> key, Content content,
Context context)
throws IOException, InterruptedException {
@@ -282,6 +278,7 @@ public class ParseSegment extends NutchTool implements Tool
{
System.exit(res);
}
+ @Override
public int run(String[] args) throws Exception {
Path segment;
@@ -312,6 +309,7 @@ public class ParseSegment extends NutchTool implements Tool
{
/*
* Used for Nutch REST service
*/
+ @Override
public Map<String, Object> run(Map<String, Object> args, String crawlId)
throws Exception {
Map<String, Object> results = new HashMap<>();
diff --git a/src/java/org/apache/nutch/parse/ParserChecker.java
b/src/java/org/apache/nutch/parse/ParserChecker.java
index e880485..8b6e3a1 100644
--- a/src/java/org/apache/nutch/parse/ParserChecker.java
+++ b/src/java/org/apache/nutch/parse/ParserChecker.java
@@ -77,6 +77,7 @@ public class ParserChecker extends AbstractChecker {
.getLogger(MethodHandles.lookup().lookupClass());
private ScoringFilters scfilters;
+ @Override
public int run(String[] args) throws Exception {
String url = null;
diff --git a/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
b/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
index 71fe42f..3c74b38 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
@@ -407,6 +407,7 @@ public class LinkDumper extends Configured implements Tool {
* Runs the LinkDumper tool. This simply creates the database, to read the
* values the nested Reader tool must be used.
*/
+ @Override
public int run(String[] args) throws Exception {
Options options = new Options();
diff --git a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
index b6bfa98..9720754 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
@@ -362,9 +362,7 @@ public class LinkRank extends Configured implements Tool {
*/
public static class CountMapper extends
Mapper<Text, Node, Text, LongWritable> {
- public void setup(Mapper<Text, Node, Text, LongWritable>.Context
context) {
- }
-
+ @Override
public void map(Text key, Node value,
Context context)
throws IOException, InterruptedException {
@@ -377,9 +375,7 @@ public class LinkRank extends Configured implements Tool {
*/
public static class CountReducer extends
Reducer<Text, LongWritable, Text, LongWritable> {
- public void setup(Reducer<Text, LongWritable, Text,
LongWritable>.Context context) {
- }
-
+ @Override
public void reduce(Text key, Iterable<LongWritable> values,
Context context)
throws IOException, InterruptedException {
@@ -642,9 +638,6 @@ public class LinkRank extends Configured implements Tool {
super(conf);
}
- public void close() {
- }
-
/**
* Runs the complete link analysis job. The complete job determins rank one
* score. Then runs through a given number of invert and analyze iterations,
@@ -736,6 +729,7 @@ public class LinkRank extends Configured implements Tool {
/**
* Runs the LinkRank tool.
*/
+ @Override
public int run(String[] args) throws Exception {
Options options = new Options();
diff --git a/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
b/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
index cc93eb8..70c4270 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
@@ -361,6 +361,7 @@ public class NodeDumper extends Configured implements Tool {
/**
* Runs the node dumper tool.
*/
+ @Override
public int run(String[] args) throws Exception {
Options options = new Options();
diff --git a/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
b/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
index 93a7c95..3674fa8 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
@@ -226,6 +226,7 @@ public class ScoreUpdater extends Configured implements
Tool{
/**
* Runs the ScoreUpdater tool.
*/
+ @Override
public int run(String[] args) throws Exception {
Options options = new Options();
diff --git a/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
b/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
index 72a6173..0b53a39 100644
--- a/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
+++ b/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
@@ -330,6 +330,7 @@ public class WebGraph extends Configured implements Tool {
/**
* Configures the OutlinkDb job reducer. Sets up internal links and link
limiting.
*/
+ @Override
public void setup(Reducer<Text, NutchWritable, Text, LinkDatum>.Context
context) {
Configuration config = context.getConfiguration();
conf = config;
@@ -340,6 +341,7 @@ public class WebGraph extends Configured implements Tool {
}
+ @Override
public void reduce(Text key, Iterable<NutchWritable> values,
Context context)
throws IOException, InterruptedException {
@@ -406,9 +408,6 @@ public class WebGraph extends Configured implements Tool {
}
}
}
-
- public void close() {
- }
}
/**
@@ -431,10 +430,12 @@ public class WebGraph extends Configured implements Tool {
* Configures job mapper. Sets timestamp for all Inlink LinkDatum
objects to the
* current system time.
*/
+ @Override
public void setup(Mapper<Text, LinkDatum, Text, LinkDatum>.Context
context) {
timestamp = System.currentTimeMillis();
}
+ @Override
public void map(Text key, LinkDatum datum,
Context context)
throws IOException, InterruptedException {
@@ -465,12 +466,7 @@ public class WebGraph extends Configured implements Tool {
public static class NodeDbReducer extends
Reducer<Text, LinkDatum, Text, Node> {
- /**
- * Configures job reducer.
- */
- public void setup(Reducer<Text, LinkDatum, Text, Node>.Context context) {
- }
-
+ @Override
public void reduce(Text key, Iterable<LinkDatum> values,
Context context) throws IOException, InterruptedException {
@@ -732,6 +728,7 @@ public class WebGraph extends Configured implements Tool {
/**
* Parses command link arguments and runs the WebGraph jobs.
*/
+ @Override
public int run(String[] args) throws Exception {
// boolean options
diff --git a/src/java/org/apache/nutch/segment/SegmentMerger.java
b/src/java/org/apache/nutch/segment/SegmentMerger.java
index 9fac5b2..a744539 100644
--- a/src/java/org/apache/nutch/segment/SegmentMerger.java
+++ b/src/java/org/apache/nutch/segment/SegmentMerger.java
@@ -363,9 +363,6 @@ public class SegmentMerger extends Configured implements
Tool{
super.setConf(conf);
}
- public void close() throws IOException {
- }
-
public static class SegmentMergerMapper extends
Mapper<Text, MetaWrapper, Text, MetaWrapper> {
@@ -755,6 +752,7 @@ public class SegmentMerger extends Configured implements
Tool{
/**
* @param args
*/
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err
diff --git a/src/java/org/apache/nutch/tools/Benchmark.java
b/src/java/org/apache/nutch/tools/Benchmark.java
index 203496b..d7c3b74 100644
--- a/src/java/org/apache/nutch/tools/Benchmark.java
+++ b/src/java/org/apache/nutch/tools/Benchmark.java
@@ -131,6 +131,7 @@ public class Benchmark extends Configured implements Tool {
}
}
+ @Override
public int run(String[] args) throws Exception {
String plugins =
"protocol-http|parse-tika|scoring-opic|urlfilter-regex|urlnormalizer-pass";
int seeds = 1;
diff --git a/src/java/org/apache/nutch/tools/FreeGenerator.java
b/src/java/org/apache/nutch/tools/FreeGenerator.java
index 4bec975..5c45903 100644
--- a/src/java/org/apache/nutch/tools/FreeGenerator.java
+++ b/src/java/org/apache/nutch/tools/FreeGenerator.java
@@ -145,6 +145,7 @@ public class FreeGenerator extends Configured implements
Tool {
}
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err
diff --git a/src/java/org/apache/nutch/tools/warc/WARCExporter.java
b/src/java/org/apache/nutch/tools/warc/WARCExporter.java
index d307000..0f06e49 100644
--- a/src/java/org/apache/nutch/tools/warc/WARCExporter.java
+++ b/src/java/org/apache/nutch/tools/warc/WARCExporter.java
@@ -90,14 +90,9 @@ public class WARCExporter extends Configured implements Tool
{
public static class WARCMapReduce {
- public void close() throws IOException {
- }
-
public static class WARCMapper extends
Mapper<Text, Writable, Text, NutchWritable> {
- public void setup(Mapper<Text, Writable, Text, NutchWritable>.Context
context) {
- }
-
+ @Override
public void map(Text key, Writable value, Context context)
throws IOException, InterruptedException {
context.write(key, new NutchWritable(value));
@@ -106,9 +101,7 @@ public class WARCExporter extends Configured implements
Tool {
public static class WARCReducer extends
Reducer<Text, NutchWritable, NullWritable, WARCWritable> {
- public void setup(Reducer<Text, NutchWritable, NullWritable,
WARCWritable>.Context context) {
- }
-
+ @Override
public void reduce(Text key, Iterable<NutchWritable> values,
Context context) throws IOException, InterruptedException {
@@ -305,6 +298,7 @@ public class WARCExporter extends Configured implements
Tool {
return 0;
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err.println(
diff --git a/src/java/org/apache/nutch/util/CrawlCompletionStats.java
b/src/java/org/apache/nutch/util/CrawlCompletionStats.java
index c138e61..f3e808b 100644
--- a/src/java/org/apache/nutch/util/CrawlCompletionStats.java
+++ b/src/java/org/apache/nutch/util/CrawlCompletionStats.java
@@ -67,6 +67,7 @@ public class CrawlCompletionStats extends Configured
implements Tool {
private static final int MODE_HOST = 1;
private static final int MODE_DOMAIN = 2;
+ @Override
public int run(String[] args) throws Exception {
Option helpOpt = new Option("h", "help", false, "Show this message");
@SuppressWarnings("static-access")
@@ -196,10 +197,12 @@ public class CrawlCompletionStats extends Configured
implements Tool {
Mapper<Text, CrawlDatum, Text, LongWritable> {
int mode = 0;
+ @Override
public void setup(Context context) {
mode = context.getConfiguration().getInt("domain.statistics.mode",
MODE_DOMAIN);
}
+ @Override
public void map(Text urlText, CrawlDatum datum, Context context)
throws IOException, InterruptedException {
@@ -225,6 +228,7 @@ public class CrawlCompletionStats extends Configured
implements Tool {
static class CrawlCompletionStatsReducer extends
Reducer<Text, LongWritable, LongWritable, Text> {
+ @Override
public void reduce(Text key, Iterable<LongWritable> values, Context
context)
throws IOException, InterruptedException {
long total = 0;
@@ -239,6 +243,7 @@ public class CrawlCompletionStats extends Configured
implements Tool {
public static class CrawlCompletionStatsCombiner extends
Reducer<Text, LongWritable, Text, LongWritable> {
+ @Override
public void reduce(Text key, Iterable<LongWritable> values, Context
context)
throws IOException, InterruptedException {
long total = 0;
diff --git a/src/java/org/apache/nutch/util/ProtocolStatusStatistics.java
b/src/java/org/apache/nutch/util/ProtocolStatusStatistics.java
index 3b6cc48..f52a9c5 100644
--- a/src/java/org/apache/nutch/util/ProtocolStatusStatistics.java
+++ b/src/java/org/apache/nutch/util/ProtocolStatusStatistics.java
@@ -64,6 +64,7 @@ public class ProtocolStatusStatistics extends Configured
implements Tool {
private static final Text UNFETCHED_TEXT = new Text("UNFETCHED");
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err.println("Usage: ProtocolStatistics inputDirs outDir
[numOfReducer]");
@@ -140,7 +141,7 @@ public class ProtocolStatusStatistics extends Configured
implements Tool {
static class ProtocolStatusStatisticsMapper extends
Mapper<Text, CrawlDatum, Text, LongWritable> {
-
+ @Override
public void map(Text urlText, CrawlDatum datum, Context context)
throws IOException, InterruptedException {
if (datum.getMetaData().containsKey(Nutch.PROTOCOL_STATUS_CODE_KEY)) {
@@ -153,6 +154,7 @@ public class ProtocolStatusStatistics extends Configured
implements Tool {
static class ProtocolStatusStatisticsReducer extends
Reducer<Text, LongWritable, LongWritable, Text> {
+ @Override
public void reduce(Text key, Iterable<LongWritable> values, Context
context)
throws IOException, InterruptedException {
long total = 0;
@@ -167,6 +169,7 @@ public class ProtocolStatusStatistics extends Configured
implements Tool {
public static class ProtocolStatusStatisticsCombiner extends
Reducer<Text, LongWritable, Text, LongWritable> {
+ @Override
public void reduce(Text key, Iterable<LongWritable> values, Context
context)
throws IOException, InterruptedException {
long total = 0;
diff --git a/src/java/org/apache/nutch/util/SitemapProcessor.java
b/src/java/org/apache/nutch/util/SitemapProcessor.java
index f558c46..aff9779 100644
--- a/src/java/org/apache/nutch/util/SitemapProcessor.java
+++ b/src/java/org/apache/nutch/util/SitemapProcessor.java
@@ -106,6 +106,7 @@ public class SitemapProcessor extends Configured implements
Tool {
private CrawlDatum datum = new CrawlDatum();
private SiteMapParser parser = null;
+ @Override
public void setup(Context context) {
Configuration conf = context.getConfiguration();
int maxSize = conf.getInt(SITEMAP_SIZE_MAX,
SiteMapParser.MAX_BYTES_ALLOWED);
@@ -127,6 +128,7 @@ public class SitemapProcessor extends Configured implements
Tool {
}
}
+ @Override
public void map(Text key, Writable value, Context context) throws
IOException, InterruptedException {
String url;
@@ -308,11 +310,13 @@ public class SitemapProcessor extends Configured
implements Tool {
private boolean overwriteExisting = false; // DO NOT ENABLE!!
+ @Override
public void setup(Context context) {
Configuration conf = context.getConfiguration();
this.overwriteExisting = conf.getBoolean(SITEMAP_OVERWRITE_EXISTING,
false);
}
+ @Override
public void reduce(Text key, Iterable<CrawlDatum> values, Context context)
throws IOException, InterruptedException {
sitemapDatum = null;
@@ -461,6 +465,7 @@ public class SitemapProcessor extends Configured implements
Tool {
System.err.println("\t-noNormalize\t\tturn off URLNormalizer on urls
(optional)");
}
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 3) {
usage();
diff --git a/src/java/org/apache/nutch/util/domain/DomainStatistics.java
b/src/java/org/apache/nutch/util/domain/DomainStatistics.java
index 4354ffc..fd2f940 100644
--- a/src/java/org/apache/nutch/util/domain/DomainStatistics.java
+++ b/src/java/org/apache/nutch/util/domain/DomainStatistics.java
@@ -62,6 +62,7 @@ public class DomainStatistics extends Configured implements
Tool {
private static final int MODE_SUFFIX = 3;
private static final int MODE_TLD = 4;
+ @Override
public int run(String[] args) throws Exception {
if (args.length < 3) {
System.err.println("Usage: DomainStatistics inputDirs outDir mode
[numOfReducer]");
@@ -161,11 +162,13 @@ public class DomainStatistics extends Configured
implements Tool {
Mapper<Text, CrawlDatum, Text, LongWritable> {
int mode = 0;
+ @Override
public void setup(Context context) {
mode = context.getConfiguration().getInt("domain.statistics.mode",
MODE_DOMAIN);
}
+ @Override
public void map(Text urlText, CrawlDatum datum, Context context)
throws IOException, InterruptedException {
@@ -209,6 +212,7 @@ public class DomainStatistics extends Configured implements
Tool {
static class DomainStatisticsReducer extends
Reducer<Text, LongWritable, LongWritable, Text> {
+ @Override
public void reduce(Text key, Iterable<LongWritable> values, Context
context)
throws IOException, InterruptedException {
long total = 0;
@@ -223,6 +227,7 @@ public class DomainStatistics extends Configured implements
Tool {
public static class DomainStatisticsCombiner extends
Reducer<Text, LongWritable, Text, LongWritable> {
+ @Override
public void reduce(Text key, Iterable<LongWritable> values, Context
context)
throws IOException, InterruptedException {
long total = 0;
diff --git
a/src/plugin/indexer-dummy/src/java/org/apache/nutch/indexwriter/dummy/DummyIndexWriter.java
b/src/plugin/indexer-dummy/src/java/org/apache/nutch/indexwriter/dummy/DummyIndexWriter.java
index d809f66..1dfc653 100644
---
a/src/plugin/indexer-dummy/src/java/org/apache/nutch/indexwriter/dummy/DummyIndexWriter.java
+++
b/src/plugin/indexer-dummy/src/java/org/apache/nutch/indexwriter/dummy/DummyIndexWriter.java
@@ -49,6 +49,7 @@ public class DummyIndexWriter implements IndexWriter {
private boolean delete = false;
private String path;
+ @Override
public void open(Configuration conf, String name) throws IOException {
//Implementation not required
}
@@ -102,6 +103,7 @@ public class DummyIndexWriter implements IndexWriter {
writer.write("add\t" + doc.getFieldValue("id") + "\n");
}
+ @Override
public void close() throws IOException {
LOG.debug("Closing dummy index file");
writer.flush();
diff --git
a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
index 86ca3eb..e8c1308 100644
---
a/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
+++
b/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/SolrIndexWriter.java
@@ -150,6 +150,7 @@ public class SolrIndexWriter implements IndexWriter {
}
}
+ @Override
public void delete(String key) throws IOException {
// escape solr hash separator
key = key.replaceAll("!", "\\!");
@@ -170,6 +171,7 @@ public class SolrIndexWriter implements IndexWriter {
write(doc);
}
+ @Override
public void write(NutchDocument doc) throws IOException {
final SolrInputDocument inputDoc = new SolrInputDocument();
@@ -201,6 +203,7 @@ public class SolrIndexWriter implements IndexWriter {
}
}
+ @Override
public void close() throws IOException {
commit();
